From e1bf2c5a392dc9dab5e87a2395c01e0bbaf6fbc4 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Tue, 29 Aug 2017 21:43:27 +0100 Subject: [PATCH] Another slight performance increase. --- .../include/CEN_patch_expert.h | 6 +- .../LandmarkDetector/src/CEN_patch_expert.cpp | 147 ++++++++++-------- .../LandmarkDetector/src/Patch_experts.cpp | 15 +- 3 files changed, 100 insertions(+), 68 deletions(-) diff --git a/lib/local/LandmarkDetector/include/CEN_patch_expert.h b/lib/local/LandmarkDetector/include/CEN_patch_expert.h index d819858b..bf4f86b8 100644 --- a/lib/local/LandmarkDetector/include/CEN_patch_expert.h +++ b/lib/local/LandmarkDetector/include/CEN_patch_expert.h @@ -79,8 +79,12 @@ namespace LandmarkDetector void Response(const cv::Mat_ &area_of_interest, cv::Mat_ &response); // Faster version of the response that only considers a subset of the area_of_interest - void ResponseSparse(const cv::Mat_ &area_of_interest, cv::Mat_ &response); + void ResponseSparse(const cv::Mat_ &area_of_interest, cv::Mat_ &response, cv::Mat_& mapMatrix); + }; + + void interpolationMatrix(cv::Mat_& mapMatrix, int response_height, int response_width, int input_width, int input_height); + } #endif diff --git a/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp b/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp index c23d7f79..0e6c419c 100644 --- a/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp +++ b/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp @@ -305,8 +305,6 @@ void im2colBiasSparseContrastNorm(const cv::Mat_& input, int width, int h { for (int i = 0; i< yB; i++) { - float* Mo = output.ptr(rowIdx); - // Skip every second row skipCounter++; if ((skipCounter + 1) % 2 == 0) @@ -314,6 +312,8 @@ void im2colBiasSparseContrastNorm(const cv::Mat_& input, int width, int h continue; } + float* Mo = output.ptr(rowIdx); + float sum = 0; for (unsigned int yy = 0; yy < height; ++yy) @@ -526,8 +526,83 @@ void im2colBiasSparse(const cv::Mat_& input, int width, int height, cv::M // response = response.t(); //} +// As the sparse patch expert output with interpolation, this function creates an interpolation matrix +void LandmarkDetector::interpolationMatrix(cv::Mat_& mapMatrix, int response_height, int response_width, int input_width, int input_height) +{ + int m = input_height; + int n = input_width; + + // determine how many blocks there will be with a sliding window of width x height in the input + int yB = m - 11 + 1; + int xB = n - 11 + 1; + + // As we will be skipping half of the outputs + int out_size = (yB*xB - 1) / 2; + + mapMatrix.create(out_size, response_height * response_width); + mapMatrix.setTo(0.0f); + + // Find a mapping from indices in the computed sparse response and the original full response + cv::Mat_ value_id_matrix(response_width, response_height, 0); + + int ind = 0; + for (int k = 0; k < value_id_matrix.rows * value_id_matrix.cols; ++k) + { + if (k % 2 != 0) + { + value_id_matrix.at(k) = ind; + ind++; + } + } + value_id_matrix = value_id_matrix.t(); + + int skip_counter = 0; + for (int x = 0; x < response_width; ++x) + { + for (int y = 0; y < response_height; ++y) + { + int mapping_col = x * response_height + y; + skip_counter++; + if (skip_counter % 2 == 0) + { + int val_id = value_id_matrix.at(y, x); + mapMatrix.at(val_id, mapping_col) = 1; + continue; + } + + double num_neigh = 0.0; + vector val_ids; + if (x - 1 >= 0) + { + num_neigh++; + val_ids.push_back(value_id_matrix.at(y, x - 1)); + } + if (y - 1 >= 0) + { + num_neigh++; + val_ids.push_back(value_id_matrix.at(y - 1, x)); + } + if (x + 1 < response_width) + { + num_neigh++; + val_ids.push_back(value_id_matrix.at(y, x + 1)); + } + if (y + 1 < response_height) + { + num_neigh++; + val_ids.push_back(value_id_matrix.at(y + 1, x)); + } + + for (size_t k = 0; k < val_ids.size(); ++k) + { + mapMatrix.at(val_ids[k], mapping_col) = 1.0 / num_neigh; + } + } + } +} + //=========================================================================== -void CEN_patch_expert::ResponseSparse(const cv::Mat_ &area_of_interest, cv::Mat_ &response) +void CEN_patch_expert::ResponseSparse(const cv::Mat_ &area_of_interest, cv::Mat_ &response, cv::Mat_& mapMatrix) { int response_height = area_of_interest.rows - height + 1; @@ -593,69 +668,9 @@ void CEN_patch_expert::ResponseSparse(const cv::Mat_ &area_of_interest, c } - // Restructure the output with interpolation - cv::Mat_ mapMatrix(response.rows, response_height * response_width, 0.0f); //TODO move this out - - // Find a mapping from indices in the computed sparse response and the original full response - cv::Mat_ value_id_matrix(response_width, response_height, 0); - - int ind = 0; - for (int k = 0; k < value_id_matrix.rows * value_id_matrix.cols; ++k) - { - if(k % 2 != 0) - { - value_id_matrix.at(k) = ind; - ind++; - } - } - value_id_matrix = value_id_matrix.t(); - - int skip_counter = 0; - for (int x = 0; x < response_width; ++x) - { - for (int y = 0; y < response_height; ++y) - { - int mapping_col = x * response_height + y; - skip_counter++; - if (skip_counter % 2 == 0) - { - int val_id = value_id_matrix.at(y, x); - mapMatrix.at(val_id, mapping_col) = 1; - continue; - } - - double num_neigh = 0.0; - vector val_ids; - if (x - 1 >= 0) - { - num_neigh++; - val_ids.push_back(value_id_matrix.at(y, x - 1)); - } - if (y - 1 >= 0) - { - num_neigh++; - val_ids.push_back(value_id_matrix.at(y - 1, x)); - } - if (x + 1 < response_width) - { - num_neigh++; - val_ids.push_back(value_id_matrix.at(y, x + 1)); - } - if (y + 1 < response_height) - { - num_neigh++; - val_ids.push_back(value_id_matrix.at(y+1, x)); - } - - for (size_t k = 0; k < val_ids.size(); ++k) - { - mapMatrix.at(val_ids[k], mapping_col) = 1.0 / num_neigh; - } - } - } - - response = response.t() * mapMatrix; - response = response.t(); + //response = response.t() * mapMatrix; + //response = response.t(); + response = mapMatrix * response; response = response.reshape(1, response_height); response = response.t(); } diff --git a/lib/local/LandmarkDetector/src/Patch_experts.cpp b/lib/local/LandmarkDetector/src/Patch_experts.cpp index 9454b163..e1f15b04 100644 --- a/lib/local/LandmarkDetector/src/Patch_experts.cpp +++ b/lib/local/LandmarkDetector/src/Patch_experts.cpp @@ -162,6 +162,19 @@ void Patch_experts::Response(vector >& patch_expert_responses, c } + // If using CEN precalculate interpolation matrix + cv::Mat_ interp_mat; + if (use_cen) + { + // Assuming the same size for all experts + int support_region = 11; + int area_of_interest_width = window_size + support_region - 1; + int area_of_interest_height = window_size + support_region - 1; + int resp_size = area_of_interest_height - support_region + 1; + interpolationMatrix(interp_mat, resp_size, resp_size, area_of_interest_width, area_of_interest_height); + interp_mat = interp_mat.t(); + } + // calculate the patch responses for every landmark, Actual work happens here. If openMP is turned on it is possible to do this in parallel, // this might work well on some machines, while potentially have an adverse effect on others #ifdef _OPENMP @@ -214,7 +227,7 @@ void Patch_experts::Response(vector >& patch_expert_responses, c // Get intensity response either from the SVR, CCNF, or CEN patch experts (prefer CEN as they are the most accurate so far) if (!cen_expert_intensity.empty()) { - cen_expert_intensity[scale][view_id][i].ResponseSparse(area_of_interest, patch_expert_responses[i]); + cen_expert_intensity[scale][view_id][i].ResponseSparse(area_of_interest, patch_expert_responses[i], interp_mat); } else if (!ccnf_expert_intensity.empty()) {