ZF/OpenFace
1
0
Fork 0
mirror of https://gitcode.com/gh_mirrors/ope/OpenFace.git synced 2026-08-31 12:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing a memory leak in Image and Sequence readers. Initial work on ability to choose face and landmark detectors in GUI. Bug fix with reporting confidence in multi-hypothesis setting. Bug fix with HAAR detector.

Browse Source
This commit is contained in:
Tadas Baltrusaitis
2018-03-31 17:02:22 +01:00
parent 959163a30f
commit b0f9b16edb
13 changed files with 184 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
gui/OpenFaceDemo/MainWindow.xaml.cs
View File

@@ -116,7 +116,7 @@ namespace OpenFaceDemo
String root = AppDomain.CurrentDomain.BaseDirectory;
// TODO, create a demo version of parameters
face_model_params = new FaceModelParameters(root, false);
face_model_params = new FaceModelParameters(root, true, false, false);
face_model_params.optimiseForVideo();
landmark_detector = new CLNF(face_model_params);
@@ -246,7 +246,8 @@ namespace OpenFaceDemo
List<Tuple<float, float>> landmarks = null;
List<Tuple<float, float>> eye_landmarks = null;
List<Tuple<Point, Point>> gaze_lines = null;
List<bool> visibilities = null;
Tuple<float, float> gaze_angle = gaze_analyser.GetGazeAngle();
if (detection_succeeding)
@@ -255,6 +256,7 @@ namespace OpenFaceDemo
eye_landmarks = landmark_detector.CalculateVisibleEyeLandmarks();
lines = landmark_detector.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
visibilities = landmark_detector.GetVisibilities();
}
// Visualisation
@@ -344,9 +346,11 @@ namespace OpenFaceDemo
eye_landmark_points.Add(new Point(p.Item1, p.Item2));
}
video.OverlayPoints.Add(landmark_points);
video.OverlayEyePoints.Add(eye_landmark_points);
video.OverlayPointsVisibility.Add(visibilities);
video.OverlayEyePoints.Add(eye_landmark_points);
video.GazeLines.Add(gaze_lines);
}

2
gui/OpenFaceDemo/OpenFaceDemo.csproj
View File

@@ -33,7 +33,7 @@
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>..\..\x64\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>

6
gui/OpenFaceOffline/MainWindow.xaml
View File

@@ -75,9 +75,9 @@
</MenuItem>
<MenuItem Header="Landmark Detector">
<MenuItem x:Name="LandmarkDetCLM" Header="CLM" IsCheckable="true" IsChecked="True"></MenuItem>
<MenuItem x:Name="LandmarkDetCLNF" Header="CLNF" IsCheckable="true"></MenuItem>
<MenuItem x:Name="LandmarkDetCECLM" Header="CE-CLM" IsCheckable="true" ></MenuItem>
<MenuItem x:Name="LandmarkDetCLM" Header="CLM" IsCheckable="true" IsChecked="{Binding LandmarkDetectorCLM}"></MenuItem>
<MenuItem x:Name="LandmarkDetCLNF" Header="CLNF" IsCheckable="true" IsChecked="{Binding LandmarkDetectorCLNF}"></MenuItem>
<MenuItem x:Name="LandmarkDetCECLM" Header="CE-CLM" IsCheckable="true" IsChecked="{Binding LandmarkDetectorCECLM}"></MenuItem>
<i:Interaction.Behaviors>
<local:ExclusiveMenuItemBehavior></local:ExclusiveMenuItemBehavior>
</i:Interaction.Behaviors>

74
gui/OpenFaceOffline/MainWindow.xaml.cs
View File

@@ -101,7 +101,7 @@ namespace OpenFaceOffline
// For tracking
FaceDetector face_detector;
FaceModelParameters face_model_params;
FaceModelParameters face_model_params; // TODO does this need to be reinitialized every time to deal with model reloading?
CLNF landmark_detector;
// For face analysis
@@ -135,6 +135,11 @@ namespace OpenFaceOffline
public bool DetectorHOG { get; set; } = false;
public bool DetectorCNN { get; set; } = true;
// Selecting which landmark detector will be used
public bool LandmarkDetectorCLM { get; set; } = false;
public bool LandmarkDetectorCLNF { get; set; } = false;
public bool LandmarkDetectorCECLM { get; set; } = true;
// For AU prediction, if videos are long dynamic models should be used
public bool DynamicAUModels { get; set; } = true;
@@ -150,12 +155,9 @@ namespace OpenFaceOffline
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
this.Icon = BitmapFrame.Create(iconUri);
// Initialize the default face detectors and landmark detectors
//as MenuItemWithRadioButton;
String root = AppDomain.CurrentDomain.BaseDirectory;
face_model_params = new FaceModelParameters(root, false);
face_model_params = new FaceModelParameters(root, LandmarkDetectorCECLM, LandmarkDetectorCLNF, LandmarkDetectorCLM);
landmark_detector = new CLNF(face_model_params);
gaze_analyser = new GazeAnalyserManaged();
@@ -192,6 +194,11 @@ namespace OpenFaceOffline
thread_running = true;
// Reload the face landmark detector if needed
ReloadLandmarkDetector();
// Set the face detector
face_model_params.SetFaceDetector(DetectorHaar, DetectorHOG, DetectorCNN);
face_model_params.optimiseForVideo();
// Setup the visualization
@@ -220,8 +227,9 @@ namespace OpenFaceOffline
var lastFrameTime = CurrentTime;
// Empty image would indicate that the stream is over
while (gray_frame.Width != 0)
while (!gray_frame.IsEmpty)
{
if(!thread_running)
{
break;
@@ -232,14 +240,14 @@ namespace OpenFaceOffline
bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(frame, face_model_params, gray_frame);
// The face analysis step (for AUs and eye gaze)
//face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, false);
//gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, false);
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
// Only the final face will contain the details
//VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), landmark_detector.GetVisibilities(), detection_succeeding, true, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress);
VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), landmark_detector.GetVisibilities(), detection_succeeding, true, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress);
// Record an observation
//RecordObservation(recorder, visualizer_of.GetVisImage(), 0, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetTimestamp(), reader.GetFrameNumber());
RecordObservation(recorder, visualizer_of.GetVisImage(), 0, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetTimestamp(), reader.GetFrameNumber());
while (thread_running & thread_paused && skip_frames == 0)
{
@@ -280,16 +288,19 @@ namespace OpenFaceOffline
// Indicate we will start running the thread
thread_running = true;
// Reload the face landmark detector if needed
ReloadLandmarkDetector();
// Setup the parameters optimized for working on individual images rather than sequences
face_model_params.optimiseForImages();
// Setup the visualization
Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance);
// Initialize the face detector if it has not been initialized yet
if (face_detector == null)
{
face_detector = new FaceDetector(face_model_params.GetMTCNNLocation());
face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation());
}
// Initialize the face analyser
@@ -330,6 +341,10 @@ namespace OpenFaceOffline
{
face_detector.DetectFacesMTCNN(face_detections, frame, confidences);
}
else if(DetectorHaar)
{
face_detector.DetectFacesHaar(face_detections, gray_frame, confidences);
}
// For visualization
double progress = reader.GetProgress();
@@ -372,6 +387,34 @@ namespace OpenFaceOffline
}
// If the landmark detector model changed need to reload it
private void ReloadLandmarkDetector()
{
bool reload = false;
if (face_model_params.IsCECLM() && !LandmarkDetectorCECLM)
{
reload = true;
}
else if(face_model_params.IsCLNF() && !LandmarkDetectorCLNF)
{
reload = true;
}
else if (face_model_params.IsCLM() && !LandmarkDetectorCLM)
{
reload = true;
}
if(reload)
{
String root = AppDomain.CurrentDomain.BaseDirectory;
face_model_params = new FaceModelParameters(root, LandmarkDetectorCECLM, LandmarkDetectorCLNF, LandmarkDetectorCLM);
landmark_detector = new CLNF(face_model_params);
}
}
private void RecordObservation(RecorderOpenFace recorder, RawImage vis_image, int face_id, bool success, float fx, float fy, float cx, float cy, double timestamp, int frame_number)
{
@@ -738,11 +781,14 @@ namespace OpenFaceOffline
StopTracking();
var image_files = openMediaDialog(true);
ImageReader reader = new ImageReader(image_files, fx, fy, cx, cy);
processing_thread = new Thread(() => ProcessIndividualImages(reader));
processing_thread.Start();
if(image_files.Count > 0)
{
ImageReader reader = new ImageReader(image_files, fx, fy, cx, cy);
processing_thread = new Thread(() => ProcessIndividualImages(reader));
processing_thread.Start();
}
}
// Selecting a directory containing images

1
lib/local/CppInerop/FaceAnalyserInterop.h
View File

@@ -275,6 +275,7 @@ public:
// May be called multiple times.
!FaceAnalyserManaged()
{
// TODO check for null pointers
delete hog_features;
delete aligned_face;
delete num_cols;

37
lib/local/CppInerop/FaceDetectorInterop.h
View File

@@ -98,12 +98,31 @@ namespace FaceDetectorInterop {
o_regions->Clear();
o_confidences->Clear();
for(size_t i = 0; i < regions_ocv.size(); ++i)
for (size_t i = 0; i < regions_ocv.size(); ++i)
{
o_regions->Add(System::Windows::Rect(regions_ocv[i].x, regions_ocv[i].y, regions_ocv[i].width, regions_ocv[i].height));
o_confidences->Add(confidences_std[i]);
}
}
// Face detection using HOG-SVM classifier
void DetectFacesHaar(List<System::Windows::Rect>^ o_regions, OpenCVWrappers::RawImage^ intensity, List<float>^ o_confidences)
{
std::vector<cv::Rect_<float> > regions_ocv;
::LandmarkDetector::DetectFaces(regions_ocv, intensity->Mat, *face_detector_haar);
o_regions->Clear();
o_confidences->Clear();
for(size_t i = 0; i < regions_ocv.size(); ++i)
{
o_regions->Add(System::Windows::Rect(regions_ocv[i].x, regions_ocv[i].y, regions_ocv[i].width, regions_ocv[i].height));
// As Haar does not provide confidence, create a fake value
o_confidences->Add(1);
}
}
// Face detection using MTCNN face detector
void DetectFacesMTCNN(List<System::Windows::Rect>^ o_regions, OpenCVWrappers::RawImage^ rgb_image, List<float>^ o_confidences)
@@ -128,10 +147,18 @@ namespace FaceDetectorInterop {
// May be called multiple times.
!FaceDetector()
{
// TODO need only basis (might be null)
delete face_detector_hog;
delete face_detector_mtcnn;
delete face_detector_haar;
if (face_detector_hog != nullptr)
{
delete face_detector_hog;
}
if (face_detector_mtcnn != nullptr)
{
delete face_detector_mtcnn;
}
if (face_detector_haar != nullptr)
{
delete face_detector_haar;
}
}
// Destructor. Called on explicit Dispose() only.

4
lib/local/CppInerop/ImageReader.h
View File

@@ -123,7 +123,7 @@ namespace UtilitiesOF {
if (m_rgb_frame == nullptr)
{
m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().width, CV_8UC3);
m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().height, CV_8UC3);
}
next_image.copyTo(m_rgb_frame->Mat);
@@ -178,7 +178,7 @@ namespace UtilitiesOF {
if (m_gray_frame == nullptr)
{
m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().width, CV_8UC3);
m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().height, CV_8UC1);
}
next_gray_image.copyTo(m_gray_frame->Mat);

64
lib/local/CppInerop/LandmarkDetectorInterop.h
View File

@@ -85,18 +85,32 @@ namespace CppInterop {
public:
// Initialise the parameters
FaceModelParameters(System::String^ root, bool demo)
FaceModelParameters(System::String^ root, bool ceclm, bool clnf, bool clm)
{
std::string root_std = msclr::interop::marshal_as<std::string>(root);
vector<std::string> args;
args.push_back(root_std);
std::string model_loc = "model/main_ceclm_general.txt";
if (ceclm)
{
model_loc = "model/main_ceclm_general.txt";
}
else if(clnf)
{
model_loc = "model/main_clnf_general.txt";
}
else if (clm)
{
model_loc = "model/main_clm_general.txt";
}
args.push_back("-mloc");
args.push_back(model_loc);
params = new ::LandmarkDetector::FaceModelParameters(args);
if(demo)
{
params->model_location = "model/main_clnf_demos.txt";
}
}
@@ -129,18 +143,54 @@ namespace CppInterop {
params->weight_factor = 0;
// Parameter optimizations for CE-CLM
if (params->is_ceclm_model)
if (params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::CECLM_DETECTOR)
{
params->sigma = 1.5f * params->sigma;
params->reg_factor = 0.9f * params->reg_factor;
}
}
bool IsCECLM()
{
return params->curr_face_detector == ::LandmarkDetector::FaceModelParameters::CECLM_DETECTOR;
}
bool IsCLNF()
{
return params->curr_face_detector == ::LandmarkDetector::FaceModelParameters::CLNF_DETECTOR;
}
bool IsCLM()
{
return params->curr_face_detector == ::LandmarkDetector::FaceModelParameters::CLM_DETECTOR;
}
System::String^ GetMTCNNLocation()
{
return gcnew System::String(params->mtcnn_face_detector_location.c_str());
}
System::String^ GetHaarLocation()
{
return gcnew System::String(params->haar_face_detector_location.c_str());
}
void SetFaceDetector(bool haar, bool hog, bool cnn)
{
if (cnn)
{
params->curr_face_detector = params->MTCNN_DETECTOR;
}
else if (hog)
{
params->curr_face_detector = params->HOG_SVM_DETECTOR;
}
else if (haar)
{
params->curr_face_detector = params->HAAR_DETECTOR;
}
}
void optimiseForImages()
{
params->window_sizes_init = vector<int>(4);
@@ -157,7 +207,7 @@ namespace CppInterop {
params->num_optimisation_iteration = 10;
// Parameter optimizations for CE-CLM
if (params->is_ceclm_model)
if (params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::MTCNN_DETECTOR)
{
params->sigma = 1.5f * params->sigma;
params->reg_factor = 0.9f * params->reg_factor;

4
lib/local/CppInerop/SequenceReader.h
View File

@@ -151,7 +151,7 @@ namespace UtilitiesOF {
if (m_rgb_frame == nullptr)
{
m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().width, CV_8UC3);
m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().height, CV_8UC3);
}
next_image.copyTo(m_rgb_frame->Mat);
@@ -221,7 +221,7 @@ namespace UtilitiesOF {
if (m_gray_frame == nullptr)
{
m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().width, CV_8UC3);
m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().height, CV_8UC1);
}
next_gray_image.copyTo(m_gray_frame->Mat);

3
lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h
View File

@@ -84,7 +84,8 @@ struct FaceModelParameters
bool multi_view;
// Based on model location, this affects the parameter settings
bool is_ceclm_model;
enum LandmarkDetector { CLM_DETECTOR, CLNF_DETECTOR, CECLM_DETECTOR };
LandmarkDetector curr_landmark_detector;
// How often should face detection be used to attempt reinitialisation, every n frames (set to negative not to reinit)
int reinit_video_every;

13
lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp
View File

@@ -268,14 +268,14 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &rgb_image, CLNF& cl
{
cv::Rect_<float> bounding_box;
// If the face detector has not been initialised read it in
if(clnf_model.face_detector_HAAR.empty())
// If the face detector has not been initialised and we're using it, then read it in
if(clnf_model.face_detector_HAAR.empty() && params.curr_face_detector == params.HAAR_DETECTOR)
{
clnf_model.face_detector_HAAR.load(params.haar_face_detector_location);
clnf_model.haar_face_detector_location = params.haar_face_detector_location;
}
if (clnf_model.face_detector_MTCNN.empty())
if (clnf_model.face_detector_MTCNN.empty() && params.curr_face_detector == params.MTCNN_DETECTOR)
{
clnf_model.face_detector_MTCNN.Read(params.mtcnn_face_detector_location);
clnf_model.mtcnn_face_detector_location = params.haar_face_detector_location;
@@ -405,6 +405,7 @@ bool DetectLandmarksInImageMultiHypBasic(const cv::Mat_<uchar> &grayscale_image,
// Store the current best estimate
float best_likelihood;
float best_detection_certainty;
cv::Vec6f best_global_parameters;
cv::Mat_<float> best_local_parameters;
cv::Mat_<float> best_detected_landmarks;
@@ -440,8 +441,9 @@ bool DetectLandmarksInImageMultiHypBasic(const cv::Mat_<uchar> &grayscale_image,
best_local_parameters = clnf_model.params_local.clone();
best_detected_landmarks = clnf_model.detected_landmarks.clone();
best_landmark_likelihoods = clnf_model.landmark_likelihoods.clone();
best_detection_certainty = clnf_model.detection_certainty;
best_success = success;
for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part)
{
best_likelihood_h[part] = clnf_model.hierarchical_models[part].model_likelihood;
@@ -461,6 +463,7 @@ bool DetectLandmarksInImageMultiHypBasic(const cv::Mat_<uchar> &grayscale_image,
clnf_model.detected_landmarks = best_detected_landmarks.clone();
clnf_model.detection_success = best_success;
clnf_model.landmark_likelihoods = best_landmark_likelihoods.clone();
clnf_model.detection_certainty = best_detection_certainty;
for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part)
{

12
lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp
View File

@@ -208,13 +208,17 @@ FaceModelParameters::FaceModelParameters(vector<string> &arguments)
if (model_path.stem().string().compare("main_ceclm_general") == 0)
{
is_ceclm_model = true;
curr_landmark_detector = CECLM_DETECTOR;
sigma = 1.5f * sigma;
reg_factor = 0.9f * reg_factor;
}
else
else if (model_path.stem().string().compare("main_clnf_general") == 0)
{
is_ceclm_model = false;
curr_landmark_detector = CLNF_DETECTOR;
}
else if (model_path.stem().string().compare("main_clm_general") == 0)
{
curr_landmark_detector = CLM_DETECTOR;
}
// Make sure face detector location is valid
@@ -296,7 +300,7 @@ void FaceModelParameters::init()
window_sizes_current = window_sizes_init;
model_location = "model/main_ceclm_general.txt";
is_ceclm_model = true;
curr_landmark_detector = CECLM_DETECTOR;
sigma = 1.5f;
reg_factor = 25.0f;

5
lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp
View File

@@ -683,7 +683,7 @@ namespace LandmarkDetector
}
// Convert from int bounding box do a double one with corrections
for (size_t face = 0; face < o_regions.size(); ++face)
for (size_t face = 0; face < face_detections.size(); ++face)
{
// OpenCV is overgenerous with face size and y location is off
// CLNF detector expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate
@@ -798,9 +798,6 @@ namespace LandmarkDetector
detector(cv_grayscale, face_detections, -0.2);
// Convert from int bounding box do a double one with corrections
//o_regions.resize(face_detections.size());
//o_confidences.resize(face_detections.size());
for (size_t face = 0; face < face_detections.size(); ++face)
{
// CLNF expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate