From f8e51bca28545d41408b622e4fdca035f7d405e4 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Fri, 1 Sep 2017 14:58:17 +0100 Subject: [PATCH] Simplification and speedup of landmark validation. --- .../include/LandmarkDetectionValidator.h | 54 +- lib/local/LandmarkDetector/src/CNN_utils.cpp | 10 +- .../src/FaceDetectorMTCNN.cpp | 2 +- .../src/LandmarkDetectionValidator.cpp | 559 ++---------------- 4 files changed, 53 insertions(+), 572 deletions(-) diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectionValidator.h b/lib/local/LandmarkDetector/include/LandmarkDetectionValidator.h index fe73fe9e..60a9c0f7 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectionValidator.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectionValidator.h @@ -83,35 +83,12 @@ class DetectionValidator public: - // What type of validator we're using - 0 - linear svr, 1 - feed forward neural net, 2 - convolutional neural net, 3 - new version of convolutional neural net - int validator_type; - // The orientations of each of the landmark detection validator vector orientations; // Piecewise affine warps to the reference shape (per orientation) vector paws; - //========================================== - // Linear SVR - - // SVR biases - vector bs; - - // SVR weights - vector > ws; - - //========================================== - // Neural Network - - // Neural net weights - vector > > ws_nn; - - // What type of activation or output functions are used - // 0 - sigmoid, 1 - tanh_opt, 2 - ReLU - vector activation_fun; - vector output_fun; - //========================================== // Convolutional Neural Network @@ -119,24 +96,19 @@ public: // view -> layer vector > > > > cnn_convolutional_layers; vector > > cnn_convolutional_layers_weights; - // Bit ugly with so much nesting, but oh well - vector > > > > > cnn_convolutional_layers_dft; - vector > > cnn_convolutional_layers_bias; + vector > > cnn_convolutional_layers_im2cold_precomp; + vector< vector > cnn_subsampling_layers; vector< vector > > cnn_fully_connected_layers_weights; - vector< vector > cnn_fully_connected_layers_bias; - // OLD CNN: 0 - convolutional, 1 - subsampling, 2 - fully connected + vector< vector > > cnn_fully_connected_layers_biases; // NEW CNN: 0 - convolutional, 1 - max pooling (2x2 stride 2), 2 - fully connected, 3 - relu, 4 - sigmoid vector > cnn_layer_types; - // Extra params for the new CNN - vector< vector > > cnn_fully_connected_layers_biases; - //========================================== // Normalisation for face validation - vector > mean_images; - vector > standard_deviations; + vector > mean_images; + vector > standard_deviations; // Default constructor DetectionValidator(){;} @@ -157,23 +129,11 @@ private: // The actual regressor application on the image - // Support Vector Regression (linear kernel) - double CheckSVR(const cv::Mat_& warped_img, int view_id); - - // Feed-forward Neural Network - double CheckNN(const cv::Mat_& warped_img, int view_id); - // Convolutional Neural Network - double DetectionValidator::CheckCNN_fast(const cv::Mat_& warped_img, int view_id); - - // Convolutional Neural Network - double CheckCNN(const cv::Mat_& warped_img, int view_id); - - // Convolutional Neural Network - double CheckCNN_old(const cv::Mat_& warped_img, int view_id); + double CheckCNN(const cv::Mat_& warped_img, int view_id); // A normalisation helper - void NormaliseWarpedToVector(const cv::Mat_& warped_img, cv::Mat_& feature_vec, int view_id); + void NormaliseWarpedToVector(const cv::Mat_& warped_img, cv::Mat_& feature_vec, int view_id); }; diff --git a/lib/local/LandmarkDetector/src/CNN_utils.cpp b/lib/local/LandmarkDetector/src/CNN_utils.cpp index 9021e76e..7346c112 100644 --- a/lib/local/LandmarkDetector/src/CNN_utils.cpp +++ b/lib/local/LandmarkDetector/src/CNN_utils.cpp @@ -542,19 +542,19 @@ namespace LandmarkDetector // Instead of re-allocating data use the first rows of already allocated data and re-allocate only if not enough rows are present im2col_multimap(input_maps, width_k, height_k, pre_alloc_im2col); - //input_matrix_mm = input_matrix_mm.t(); - cv::Mat_ weight_t = weight_matrix.t(); float* m1 = (float*)pre_alloc_im2col.data; - float* m2 = (float*)weight_t.data; + float* m2 = (float*)weight_matrix.data; + int m2_cols = weight_matrix.cols; + int m2_rows = weight_matrix.rows; - cv::Mat_ out(num_rows, weight_t.cols, 1.0); + cv::Mat_ out(num_rows, weight_matrix.cols, 1.0); float* m3 = (float*)out.data; //cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, weight_t.cols, yB * xB, pre_alloc_im2col.cols, 1, m2, weight_t.cols, m1, pre_alloc_im2col.cols, 0.0, m3, weight_t.cols); float alpha = 1.0f; float beta = 0.0f; // Call fortran directly (faster) - sgemm_("N", "N", &weight_t.cols, &num_rows, &pre_alloc_im2col.cols, &alpha, m2, &weight_t.cols, m1, &pre_alloc_im2col.cols, &beta, m3, &weight_t.cols); + sgemm_("N", "N", &m2_cols, &num_rows, &pre_alloc_im2col.cols, &alpha, m2, &m2_cols, m1, &pre_alloc_im2col.cols, &beta, m3, &m2_cols); out = out.t(); // Move back to vectors and reshape accordingly diff --git a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp index b2fe42c0..7e9e7aa6 100644 --- a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp +++ b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp @@ -482,7 +482,7 @@ void CNN::Read(const string& location) } weight_matrix.copyTo(W(cv::Rect(0, 0, weight_matrix.cols, weight_matrix.rows))); - cnn_convolutional_layers_weights.push_back(W); + cnn_convolutional_layers_weights.push_back(W.t()); conv_layer_pre_alloc_im2col.push_back(cv::Mat_()); } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp index d40cd997..d6dad931 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp @@ -84,35 +84,10 @@ using namespace LandmarkDetector; // Copy constructor -DetectionValidator::DetectionValidator(const DetectionValidator& other) : orientations(other.orientations), bs(other.bs), paws(other.paws), -cnn_subsampling_layers(other.cnn_subsampling_layers), cnn_layer_types(other.cnn_layer_types), cnn_fully_connected_layers_bias(other.cnn_fully_connected_layers_bias), -cnn_convolutional_layers_bias(other.cnn_convolutional_layers_bias), cnn_convolutional_layers_dft(other.cnn_convolutional_layers_dft) +DetectionValidator::DetectionValidator(const DetectionValidator& other) : orientations(other.orientations), paws(other.paws), +cnn_subsampling_layers(other.cnn_subsampling_layers), cnn_layer_types(other.cnn_layer_types), cnn_convolutional_layers_im2cold_precomp(other.cnn_convolutional_layers_im2cold_precomp) { - this->validator_type = other.validator_type; - - this->activation_fun = other.activation_fun; - this->output_fun = other.output_fun; - - this->ws.resize(other.ws.size()); - for (size_t i = 0; i < other.ws.size(); ++i) - { - // Make sure the matrix is copied. - this->ws[i] = other.ws[i].clone(); - } - - this->ws_nn.resize(other.ws_nn.size()); - for (size_t i = 0; i < other.ws_nn.size(); ++i) - { - this->ws_nn[i].resize(other.ws_nn[i].size()); - - for (size_t k = 0; k < other.ws_nn[i].size(); ++k) - { - // Make sure the matrix is copied. - this->ws_nn[i][k] = other.ws_nn[i][k].clone(); - } - } - this->cnn_convolutional_layers.resize(other.cnn_convolutional_layers.size()); for (size_t v = 0; v < other.cnn_convolutional_layers.size(); ++v) { @@ -187,8 +162,14 @@ void DetectionValidator::Read(string location) detection_validator_stream.seekg (0, ios::beg); // Read validator type + int validator_type; detection_validator_stream.read ((char*)&validator_type, 4); - + + if (validator_type != 3) + { + cout << "ERROR: Using old face validator, no longer supported" << endl; + } + // Read the number of views (orientations) within the validator int n; detection_validator_stream.read ((char*)&n, 4); @@ -209,40 +190,12 @@ void DetectionValidator::Read(string location) // Initialise the piece-wise affine warps, biases and weights paws.resize(n); - if( validator_type == 0) - { - // Reading in SVRs - bs.resize(n); - ws.resize(n); - } - else if(validator_type == 1) - { - // Reading in NNs - ws_nn.resize(n); - - activation_fun.resize(n); - output_fun.resize(n); - } - else if(validator_type == 2) - { - cnn_convolutional_layers.resize(n); - cnn_convolutional_layers_dft.resize(n); - cnn_subsampling_layers.resize(n); - cnn_fully_connected_layers_weights.resize(n); - cnn_layer_types.resize(n); - cnn_fully_connected_layers_bias.resize(n); - cnn_convolutional_layers_bias.resize(n); - } - else if (validator_type == 3) - { - cnn_convolutional_layers_weights.resize(n); - cnn_convolutional_layers.resize(n); - cnn_convolutional_layers_dft.resize(n); - cnn_fully_connected_layers_weights.resize(n); - cnn_layer_types.resize(n); - cnn_fully_connected_layers_biases.resize(n); - cnn_convolutional_layers_bias.resize(n); - } + cnn_convolutional_layers_weights.resize(n); + cnn_convolutional_layers_im2cold_precomp.resize(n); + cnn_convolutional_layers.resize(n); + cnn_fully_connected_layers_weights.resize(n); + cnn_layer_types.resize(n); + cnn_fully_connected_layers_biases.resize(n); // Initialise the normalisation terms mean_images.resize(n); @@ -253,126 +206,19 @@ void DetectionValidator::Read(string location) { // Read in the mean images - LandmarkDetector::ReadMatBin(detection_validator_stream, mean_images[i]); + cv::Mat_ mean_img; + LandmarkDetector::ReadMatBin(detection_validator_stream, mean_img); + mean_img.convertTo(mean_images[i], CV_32F); mean_images[i] = mean_images[i].t(); - - LandmarkDetector::ReadMatBin(detection_validator_stream, standard_deviations[i]); + + cv::Mat_ std_dev; + LandmarkDetector::ReadMatBin(detection_validator_stream, std_dev); + std_dev.convertTo(standard_deviations[i], CV_32F); + standard_deviations[i] = standard_deviations[i].t(); // Model specifics - if(validator_type == 0) - { - // Reading in the biases and weights - detection_validator_stream.read ((char*)&bs[i], 8); - LandmarkDetector::ReadMatBin(detection_validator_stream, ws[i]); - - } - else if(validator_type == 1) - { - - // Reading in the number of layers in the neural net - int num_depth_layers; - detection_validator_stream.read ((char*)&num_depth_layers, 4); - - // Reading in activation and output function types - detection_validator_stream.read ((char*)&activation_fun[i], 4); - detection_validator_stream.read ((char*)&output_fun[i], 4); - - ws_nn[i].resize(num_depth_layers); - for(int layer = 0; layer < num_depth_layers; layer++) - { - LandmarkDetector::ReadMatBin(detection_validator_stream, ws_nn[i][layer]); - - // Transpose for efficiency during multiplication - ws_nn[i][layer] = ws_nn[i][layer].t(); - } - } - else if(validator_type == 2) - { - // Reading in CNNs - - int network_depth; - detection_validator_stream.read ((char*)&network_depth, 4); - - cnn_layer_types[i].resize(network_depth); - - for(int layer = 0; layer < network_depth; ++layer) - { - - int layer_type; - detection_validator_stream.read ((char*)&layer_type, 4); - cnn_layer_types[i][layer] = layer_type; - - // convolutional - if(layer_type == 0) - { - - // Read the number of input maps - int num_in_maps; - detection_validator_stream.read ((char*)&num_in_maps, 4); - - // Read the number of kernels for each input map - int num_kernels; - detection_validator_stream.read ((char*)&num_kernels, 4); - - vector > > kernels; - vector > > > kernel_dfts; - - kernels.resize(num_in_maps); - kernel_dfts.resize(num_in_maps); - - vector biases; - for (int k = 0; k < num_kernels; ++k) - { - float bias; - detection_validator_stream.read ((char*)&bias, 4); - biases.push_back(bias); - } - - cnn_convolutional_layers_bias[i].push_back(biases); - - // For every input map - for (int in = 0; in < num_in_maps; ++in) - { - kernels[in].resize(num_kernels); - kernel_dfts[in].resize(num_kernels); - - // For every kernel on that input map - for (int k = 0; k < num_kernels; ++k) - { - ReadMatBin(detection_validator_stream, kernels[in][k]); - - // Flip the kernel in order to do convolution and not correlation - cv::flip(kernels[in][k], kernels[in][k], -1); - } - } - - cnn_convolutional_layers[i].push_back(kernels); - cnn_convolutional_layers_dft[i].push_back(kernel_dfts); - } - else if(layer_type == 1) - { - // Subsampling layer - - int scale; - detection_validator_stream.read ((char*)&scale, 4); - - cnn_subsampling_layers[i].push_back(scale); - } - else if(layer_type == 2) - { - float bias; - detection_validator_stream.read ((char*)&bias, 4); - cnn_fully_connected_layers_bias[i].push_back(bias); - - // Fully connected layer - cv::Mat_ weights; - ReadMatBin(detection_validator_stream, weights); - cnn_fully_connected_layers_weights[i].push_back(weights); - } - } - } - else if (validator_type == 3) + if (validator_type == 3) { int network_depth; detection_validator_stream.read((char*)&network_depth, 4); @@ -399,10 +245,8 @@ void DetectionValidator::Read(string location) detection_validator_stream.read((char*)&num_kernels, 4); vector > > kernels; - vector > > > kernel_dfts; kernels.resize(num_in_maps); - kernel_dfts.resize(num_in_maps); vector biases; for (int k = 0; k < num_kernels; ++k) @@ -412,13 +256,10 @@ void DetectionValidator::Read(string location) biases.push_back(bias); } - cnn_convolutional_layers_bias[i].push_back(biases); - // For every input map for (int in = 0; in < num_in_maps; ++in) { kernels[in].resize(num_kernels); - kernel_dfts[in].resize(num_kernels); // For every kernel on that input map for (int k = 0; k < num_kernels; ++k) @@ -429,7 +270,6 @@ void DetectionValidator::Read(string location) } cnn_convolutional_layers[i].push_back(kernels); - cnn_convolutional_layers_dft[i].push_back(kernel_dfts); // Rearrange the kernels for faster inference with FFT vector > > kernels_rearr; @@ -468,7 +308,8 @@ void DetectionValidator::Read(string location) } weight_matrix.copyTo(W(cv::Rect(0, 0, weight_matrix.cols, weight_matrix.rows))); - cnn_convolutional_layers_weights[i].push_back(W); + cnn_convolutional_layers_weights[i].push_back(W.t()); + cnn_convolutional_layers_im2cold_precomp[i].push_back(cv::Mat_()); } else if (layer_type == 2) { @@ -502,342 +343,22 @@ double DetectionValidator::Check(const cv::Vec3d& orientation, const cv::Mat_ warped; + cv::Mat_ warped; // the piece-wise affine image - cv::Mat_ intensity_img_double; - intensity_img.convertTo(intensity_img_double, CV_64F); + cv::Mat_ intensity_img_float; + intensity_img.convertTo(intensity_img_float, CV_32F); - paws[id].Warp(intensity_img_double, warped, detected_landmarks); + paws[id].Warp(intensity_img_float, warped, detected_landmarks); - double dec; - if(validator_type == 0) - { - dec = CheckSVR(warped, id); - } - else if(validator_type == 1) - { - dec = CheckNN(warped, id); - } - else if(validator_type == 2) - { - dec = CheckCNN_old(warped, id); - } - else if (validator_type == 3) - { - - dec = CheckCNN(warped, id); - } + double dec = CheckCNN(warped, id); return dec; } -double DetectionValidator::CheckNN(const cv::Mat_& warped_img, int view_id) -{ - cv::Mat_ feature_vec; - NormaliseWarpedToVector(warped_img, feature_vec, view_id); - feature_vec = feature_vec.t(); - - for(size_t layer = 0; layer < ws_nn[view_id].size(); ++layer) - { - // Add a bias term - cv::hconcat(cv::Mat_(1,1, 1.0), feature_vec, feature_vec); - - // Apply the weights - feature_vec = feature_vec * ws_nn[view_id][layer]; - - // Activation or output - int fun_type; - if(layer != ws_nn[view_id].size() - 1) - { - fun_type = activation_fun[view_id]; - } - else - { - fun_type = output_fun[view_id]; - } - - if(fun_type == 0) - { - cv::exp(-feature_vec, feature_vec); - feature_vec = 1.0 /(1.0 + feature_vec); - } - else if(fun_type == 1) - { - cv::MatIterator_ q1 = feature_vec.begin(); // respone for each pixel - cv::MatIterator_ q2 = feature_vec.end(); - - // the logistic function (sigmoid) applied to the response - while(q1 != q2) - { - *q1 = 1.7159 * tanh((2.0/3.0) * (*q1)); - q1++; - } - } - // TODO ReLU - - } - - // Turn it to -1, 1 range - double dec = (feature_vec.at(0) - 0.5) * 2; - - return dec; - -} - -double DetectionValidator::CheckSVR(const cv::Mat_& warped_img, int view_id) +double DetectionValidator::CheckCNN(const cv::Mat_& warped_img, int view_id) { - cv::Mat_ feature_vec; - NormaliseWarpedToVector(warped_img, feature_vec, view_id); - - - double dec = (ws[view_id].dot(feature_vec.t()) + bs[view_id]); - - return dec; - -} - -// Convolutional Neural Network -double DetectionValidator::CheckCNN_old(const cv::Mat_& warped_img, int view_id) -{ - - cv::Mat_ feature_vec; - NormaliseWarpedToVector(warped_img, feature_vec, view_id); - - // Create a normalised image from the crop vector - cv::Mat_ img(warped_img.size(), 0.0); - img = img.t(); - - cv::Mat mask = paws[view_id].pixel_mask.t(); - cv::MatIterator_ mask_it = mask.begin(); - - cv::MatIterator_ feature_it = feature_vec.begin(); - cv::MatIterator_ img_it = img.begin(); - - int wInt = img.cols; - int hInt = img.rows; - - for(int i=0; i < wInt; ++i) - { - for(int j=0; j < hInt; ++j, ++mask_it, ++img_it) - { - // if is within mask - if(*mask_it) - { - // assign the feature to image if it is within the mask - *img_it = (float)*feature_it++; - } - } - } - img = img.t(); - - int cnn_layer = 0; - int subsample_layer = 0; - int fully_connected_layer = 0; - - vector > input_maps; - input_maps.push_back(img); - - vector > outputs; - - for(size_t layer = 0; layer < cnn_layer_types[view_id].size(); ++layer) - { - // Determine layer type - int layer_type = cnn_layer_types[view_id][layer]; - - - // Convolutional layer - if(layer_type == 0) - { - vector > outputs_kern; - for(size_t in = 0; in < input_maps.size(); ++in) - { - cv::Mat_ input_image = input_maps[in]; - - // Useful precomputed data placeholders for quick correlation (convolution) - cv::Mat_ input_image_dft; - cv::Mat integral_image; - cv::Mat integral_image_sq; - - for(size_t k = 0; k < cnn_convolutional_layers[view_id][cnn_layer][in].size(); ++k) - { - cv::Mat_ kernel = cnn_convolutional_layers[view_id][cnn_layer][in][k]; - - // The convolution (with precomputation) - cv::Mat_ output; - if(cnn_convolutional_layers_dft[view_id][cnn_layer][in][k].second.empty()) - { - std::map > precomputed_dft; - - LandmarkDetector::matchTemplate_m(input_image, input_image_dft, integral_image, integral_image_sq, kernel, precomputed_dft, output, CV_TM_CCORR); - - cnn_convolutional_layers_dft[view_id][cnn_layer][in][k].first = precomputed_dft.begin()->first; - cnn_convolutional_layers_dft[view_id][cnn_layer][in][k].second = precomputed_dft.begin()->second; - } - else - { - std::map > precomputed_dft; - precomputed_dft[cnn_convolutional_layers_dft[view_id][cnn_layer][in][k].first] = cnn_convolutional_layers_dft[view_id][cnn_layer][in][k].second; - LandmarkDetector::matchTemplate_m(input_image, input_image_dft, integral_image, integral_image_sq, kernel, precomputed_dft, output, CV_TM_CCORR); - } - - // Combining the maps - if(in == 0) - { - outputs_kern.push_back(output); - } - else - { - outputs_kern[k] = outputs_kern[k] + output; - } - - } - - } - - outputs.clear(); - for(size_t k = 0; k < cnn_convolutional_layers[view_id][cnn_layer][0].size(); ++k) - { - // Apply the sigmoid - cv::exp(-outputs_kern[k] - cnn_convolutional_layers_bias[view_id][cnn_layer][k], outputs_kern[k]); - outputs_kern[k] = 1.0 /(1.0 + outputs_kern[k]); - - outputs.push_back(outputs_kern[k]); - - } - - cnn_layer++; - } - if(layer_type == 1) - { - // Subsampling layer - int scale = cnn_subsampling_layers[view_id][subsample_layer]; - - cv::Mat kx = cv::Mat::ones(2, 1, CV_32F)*1.0f/scale; - cv::Mat ky = cv::Mat::ones(1, 2, CV_32F)*1.0f/scale; - - vector> outputs_sub; - for(size_t in = 0; in < input_maps.size(); ++in) - { - - cv::Mat_ conv_out; - - cv::sepFilter2D(input_maps[in], conv_out, CV_32F, kx, ky); - conv_out = conv_out(cv::Rect(1, 1, conv_out.cols - 1, conv_out.rows - 1)); - - int res_rows = conv_out.rows / scale; - int res_cols = conv_out.cols / scale; - - if(conv_out.rows % scale != 0) - { - res_rows++; - } - if(conv_out.cols % scale != 0) - { - res_cols++; - } - - cv::Mat_ sub_out(res_rows, res_cols); - for(int w = 0; w < conv_out.cols; w+=scale) - { - for(int h=0; h < conv_out.rows; h+=scale) - { - sub_out.at(h/scale, w/scale) = conv_out(h, w); - } - } - outputs_sub.push_back(sub_out); - } - outputs = outputs_sub; - subsample_layer++; - - } - if(layer_type == 2) - { - // Concatenate all the maps - cv::Mat_ input_concat = input_maps[0].t(); - input_concat = input_concat.reshape(0, 1); - - for(size_t in = 1; in < input_maps.size(); ++in) - { - cv::Mat_ add = input_maps[in].t(); - add = add.reshape(0,1); - cv::hconcat(input_concat, add, input_concat); - } - - input_concat = input_concat * cnn_fully_connected_layers_weights[view_id][fully_connected_layer].t(); - - cv::exp(-input_concat - cnn_fully_connected_layers_bias[view_id][fully_connected_layer], input_concat); - input_concat = 1.0 /(1.0 + input_concat); - - outputs.clear(); - outputs.push_back(input_concat); - - fully_connected_layer++; - } - // Max pooling layer - if (layer_type == 3) - { - - vector> outputs_sub; - - // Iterate over pool height and width, all the stride is 2x2 and no padding is used - int stride_x = 2; - int stride_y = 2; - - int pool_x = 2; - int pool_y = 2; - - for (size_t in = 0; in < input_maps.size(); ++in) - { - int out_x = input_maps[in].cols / stride_x; - int out_y = input_maps[in].rows / stride_y; - - cv::Mat_ sub_out(out_y, out_x, 0.0); - cv::Mat_ in_map = input_maps[in]; - - for (int x = 0; x < input_maps[in].cols; x+= stride_x) - { - for (int y = 0; y < input_maps[in].rows; y+= stride_y) - { - float curr_max = -FLT_MAX; - for (int x_in = x; x_in < x+pool_x; ++x_in) - { - for (int y_in = y; y_in < y + pool_y; ++y_in) - { - float curr_val = in_map.at(y_in, x_in); - if (curr_val > curr_max) - { - curr_max = curr_val; - } - } - } - int x_in_out = x / stride_x; - int y_in_out = y / stride_y; - sub_out.at(y_in_out, x_in_out) = curr_max; - } - } - - outputs_sub.push_back(sub_out); - } - outputs = outputs_sub; - subsample_layer++; - } - - // Set the outputs of this layer to inputs of the next - input_maps = outputs; - - } - - // Turn it to -1, 1 range - double dec = (outputs[0].at(0) - 0.5) * 2.0; - - return dec; -} - -double DetectionValidator::CheckCNN(const cv::Mat_& warped_img, int view_id) -{ - - cv::Mat_ feature_vec; + cv::Mat_ feature_vec; NormaliseWarpedToVector(warped_img, feature_vec, view_id); // Create a normalised image from the crop vector @@ -847,7 +368,7 @@ double DetectionValidator::CheckCNN(const cv::Mat_& warped_img, int view cv::Mat mask = paws[view_id].pixel_mask.t(); cv::MatIterator_ mask_it = mask.begin(); - cv::MatIterator_ feature_it = feature_vec.begin(); + cv::MatIterator_ feature_it = feature_vec.begin(); cv::MatIterator_ img_it = img.begin(); int wInt = img.cols; @@ -884,7 +405,7 @@ double DetectionValidator::CheckCNN(const cv::Mat_& warped_img, int view if (layer_type == 0) { - convolution_direct_blas(outputs, input_maps, cnn_convolutional_layers_weights[view_id][cnn_layer], cnn_convolutional_layers[view_id][cnn_layer][0][0].rows, cnn_convolutional_layers[view_id][cnn_layer][0][0].cols); + convolution_direct_blas_nts(outputs, input_maps, cnn_convolutional_layers_weights[view_id][cnn_layer], cnn_convolutional_layers[view_id][cnn_layer][0][0].rows, cnn_convolutional_layers[view_id][cnn_layer][0][0].cols, cnn_convolutional_layers_im2cold_precomp[view_id][cnn_layer]); cnn_layer++; } @@ -942,15 +463,15 @@ double DetectionValidator::CheckCNN(const cv::Mat_& warped_img, int view return unquantized; } -void DetectionValidator::NormaliseWarpedToVector(const cv::Mat_& warped_img, cv::Mat_& feature_vec, int view_id) +void DetectionValidator::NormaliseWarpedToVector(const cv::Mat_& warped_img, cv::Mat_& feature_vec, int view_id) { - cv::Mat_ warped_t = warped_img.t(); + cv::Mat_ warped_t = warped_img.t(); // the vector to be filled with paw values - cv::MatIterator_ vp; - cv::MatIterator_ cp; + cv::MatIterator_ vp; + cv::MatIterator_ cp; - cv::Mat_ vec(paws[view_id].number_of_pixels,1); + cv::Mat_ vec(paws[view_id].number_of_pixels,1); vp = vec.begin(); cp = warped_t.begin();