ZF/OpenFace
1
0
Fork 0
mirror of https://gitcode.com/gh_mirrors/ope/OpenFace.git synced 2026-05-16 20:28:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bringing CE-CLM up to speed with master.

Browse Source
This commit is contained in:
Tadas Baltrusaitis
2017-08-03 11:14:49 -04:00
parent af77a045d9 cdd258330f
commit f19fab7ff7
234 changed files with 1852 additions and 16663 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
lib/local/LandmarkDetector/src/Patch_experts.cpp
View File

@@ -1,38 +1,14 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge,
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// THIS SOFTWARE IS PROVIDED <20>AS IS<49> FOR ACADEMIC USE ONLY AND ANY EXPRESS
// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS
// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY.
// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// Notwithstanding the license granted herein, Licensee acknowledges that certain components
// of the Software may be covered by so-called <20>open source<63> software licenses (<28>Open Source
// Components<74>), which means any software licenses approved as open source licenses by the
// Open Source Initiative or any substantially similar licenses, including without limitation any
// license that, as a condition of distribution of the software licensed under such license,
// requires that the distributor make the software available in source code format. Licensor shall
// provide a list of Open Source Components for a particular version of the Software upon
// Licensee<65>s request. Licensee will comply with the applicable terms of such licenses and to
// the extent required by the licenses covering Open Source Components, the terms of such
// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the
// licenses applicable to Open Source Components prohibit any of the restrictions in this
// License Agreement with respect to such Open Source Component, such restrictions will not
// apply to such Open Source Component. To the extent the terms of the licenses applicable to
// Open Source Components require Licensor to make an offer to provide source code or
// related information in connection with the Software, such offer is hereby made. Any request
// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk
// Licensee acknowledges receipt of notices for the Open Source Components for the initial
// delivery of the Software.
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
@@ -80,8 +56,9 @@
using namespace LandmarkDetector;
// A copy constructor
Patch_experts::Patch_experts(const Patch_experts& other) : patch_scaling(other.patch_scaling), centers(other.centers), svr_expert_intensity(other.svr_expert_intensity),
svr_expert_depth(other.svr_expert_depth), ccnf_expert_intensity(other.ccnf_expert_intensity), cen_expert_intensity(other.cen_expert_intensity),
ccnf_expert_intensity(other.ccnf_expert_intensity), cen_expert_intensity(other.cen_expert_intensity),
early_term_weights(other.early_term_weights), early_term_biases(other.early_term_biases), early_term_cutoffs(other.early_term_cutoffs)
{
@@ -112,11 +89,11 @@ Patch_experts::Patch_experts(const Patch_experts& other) : patch_scaling(other.p
}
}
// Returns the patch expert responses given a grayscale and an optional depth image.
// Returns the patch expert responses given a grayscale image.
// Additionally returns the transform from the image coordinates to the response coordinates (and vice versa).
// The computation also requires the current landmark locations to compute response around, the PDM corresponding to the desired model, and the parameters describing its instance
// Also need to provide the size of the area of interest and the desired scale of analysis
void Patch_experts::Response(vector<cv::Mat_<float> >& patch_expert_responses, cv::Matx22f& sim_ref_to_img, cv::Matx22d& sim_img_to_ref, const cv::Mat_<uchar>& grayscale_image, const cv::Mat_<float>& depth_image,
void Patch_experts::Response(vector<cv::Mat_<float> >& patch_expert_responses, cv::Matx22f& sim_ref_to_img, cv::Matx22d& sim_img_to_ref, const cv::Mat_<uchar>& grayscale_image,
const PDM& pdm, const cv::Vec6d& params_global, const cv::Mat_<double>& params_local, int window_size, int scale)
{
@@ -152,15 +129,6 @@ void Patch_experts::Response(vector<cv::Mat_<float> >& patch_expert_responses, c
sim_ref_to_img(1,0) = (float)sim_ref_to_img_d(1,0);
sim_ref_to_img(1,1) = (float)sim_ref_to_img_d(1,1);
// Indicates the legal pixels in a depth image, if available (used for CLM-Z area of interest (window) interpolation)
cv::Mat_<uchar> mask;
if(!depth_image.empty())
{
mask = depth_image > 0;
mask = mask / 255;
}
bool use_ccnf = !this->ccnf_expert_intensity.empty();
bool use_cen = !this->cen_expert_intensity.empty();
@@ -264,54 +232,6 @@ void Patch_experts::Response(vector<cv::Mat_<float> >& patch_expert_responses, c
{
svr_expert_intensity[scale][view_id][i].Response(area_of_interest, patch_expert_responses[i]);
}
// if we have a corresponding depth patch and it is visible
if (!svr_expert_depth.empty() && !depth_image.empty() && visibilities[scale][view_id].at<int>(i, 0))
{
cv::Mat_<float> dProb = patch_expert_responses[i].clone();
cv::Mat_<float> depthWindow(area_of_interest_height, area_of_interest_width);
CvMat dimg_o = depthWindow;
cv::Mat maskWindow(area_of_interest_height, area_of_interest_width, CV_32F);
CvMat mimg_o = maskWindow;
IplImage d_o = depth_image;
IplImage m_o = mask;
cvGetQuadrangleSubPix(&d_o, &dimg_o, &sim_o);
cvGetQuadrangleSubPix(&m_o, &mimg_o, &sim_o);
depthWindow.setTo(0, maskWindow < 1);
svr_expert_depth[scale][view_id][i].ResponseDepth(depthWindow, dProb);
// Sum to one
double sum = cv::sum(patch_expert_responses[i])[0];
// To avoid division by 0 issues
if (sum == 0)
{
sum = 1;
}
patch_expert_responses[i] /= sum;
// Sum to one
sum = cv::sum(dProb)[0];
// To avoid division by 0 issues
if (sum == 0)
{
sum = 1;
}
dProb /= sum;
patch_expert_responses[i] = patch_expert_responses[i] + dProb;
}
}
}
}
@@ -346,7 +266,7 @@ int Patch_experts::GetViewIdx(const cv::Vec6d& params_global, int scale) const
//===========================================================================
void Patch_experts::Read(vector<string> intensity_svr_expert_locations, vector<string> depth_svr_expert_locations, vector<string> intensity_ccnf_expert_locations, vector<string> intensity_cen_expert_locations, string early_term_loc)
void Patch_experts::Read(vector<string> intensity_svr_expert_locations, vector<string> intensity_ccnf_expert_locations, vector<string> intensity_cen_expert_locations, string early_term_loc)
{
// initialise the SVR intensity patch expert parameters
@@ -403,59 +323,6 @@ void Patch_experts::Read(vector<string> intensity_svr_expert_locations, vector<s
Read_CEN_patch_experts(location, centers[scale], visibilities[scale], cen_expert_intensity[scale], patch_scaling[scale]);
}
// initialise the SVR depth patch expert parameters
int num_depth_scales = depth_svr_expert_locations.size();
int num_intensity_scales = centers.size();
if(num_depth_scales > 0 && num_intensity_scales != num_depth_scales)
{
cout << "Intensity and depth patch experts have a different number of scales, can't read depth" << endl;
return;
}
// Have these to confirm that depth patch experts have the same number of views and scales and have the same visibilities
vector<vector<cv::Vec3d> > centers_depth(num_depth_scales);
vector<vector<cv::Mat_<int> > > visibilities_depth(num_depth_scales);
vector<double> patch_scaling_depth(num_depth_scales);
svr_expert_depth.resize(num_depth_scales);
// Reading in SVR intensity patch experts for each scales it is defined in
for(int scale = 0; scale < num_depth_scales; ++scale)
{
string location = depth_svr_expert_locations[scale];
cout << "Reading the depth SVR patch experts from: " << location << "....";
Read_SVR_patch_experts(location, centers_depth[scale], visibilities_depth[scale], svr_expert_depth[scale], patch_scaling_depth[scale]);
// Check if the scales are identical
if(patch_scaling_depth[scale] != patch_scaling[scale])
{
cout << "Intensity and depth patch experts have a different scales, can't read depth" << endl;
svr_expert_depth.clear();
return;
}
int num_views_intensity = centers[scale].size();
int num_views_depth = centers_depth[scale].size();
// Check if the number of views is identical
if(num_views_intensity != num_views_depth)
{
cout << "Intensity and depth patch experts have a different number of scales, can't read depth" << endl;
svr_expert_depth.clear();
return;
}
for(int view = 0; view < num_views_depth; ++view)
{
if(cv::countNonZero(centers_depth[scale][view] != centers[scale][view]) || cv::countNonZero(visibilities[scale][view] != visibilities_depth[scale][view]))
{
cout << "Intensity and depth patch experts have different visibilities or centers" << endl;
svr_expert_depth.clear();
return;
}
}
}
// Reading in early termination parameters
if (!early_term_loc.empty())