From 3c5dad78256a2abf7fd3744caa0372c64df02a32 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Wed, 2 May 2018 08:06:37 +0100 Subject: [PATCH] Adding ability to not use MTCNN if the model is not present in GUI. --- .gitignore | 1 + exe/FaceLandmarkImg/FaceLandmarkImg.cpp | 2 +- gui/HeadPose-live/MainWindow.xaml.cs | 15 ++++++++++++- gui/OpenFaceDemo/MainWindow.xaml.cs | 10 +++++++++ gui/OpenFaceOffline/MainWindow.xaml | 23 ++++++-------------- gui/OpenFaceOffline/MainWindow.xaml.cs | 26 ++++++++++++++++++++++- lib/local/CppInerop/FaceDetectorInterop.h | 5 +++++ 7 files changed, 63 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index e63dfe49..a3b4aa78 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,4 @@ gui/OpenFaceOffline/obj/ gui/OpenFaceDemo/obj/ exe/FeatureExtraction/processed/ matlab_runners/ +exe/FaceLandmarkVidMulti/processed/ diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index db8c6e87..3e0a386f 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -233,7 +233,6 @@ int main(int argc, char **argv) // Setting up the recorder output open_face_rec.SetObservationHOG(face_model.detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG - open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy), face_model.params_global, face_model.params_local, face_model.detection_certainty, face_model.detection_success); @@ -249,6 +248,7 @@ int main(int argc, char **argv) visualizer.ShowObservation(); } + open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); open_face_rec.WriteObservationTracked(); // Grabbing the next frame in the sequence diff --git a/gui/HeadPose-live/MainWindow.xaml.cs b/gui/HeadPose-live/MainWindow.xaml.cs index ad030f5d..e45e6212 100644 --- a/gui/HeadPose-live/MainWindow.xaml.cs +++ b/gui/HeadPose-live/MainWindow.xaml.cs @@ -48,6 +48,7 @@ using CppInterop; using CppInterop.LandmarkDetector; using System.Windows.Threading; using GazeAnalyser_Interop; +using FaceDetectorInterop; using ZeroMQ; using System.Drawing; @@ -370,6 +371,16 @@ namespace HeadPoseLive String root = AppDomain.CurrentDomain.BaseDirectory; FaceModelParameters model_params = new FaceModelParameters(root, true, false, false); + + // Initialize the face detector + FaceDetector face_detector = new FaceDetector(model_params.GetHaarLocation(), model_params.GetMTCNNLocation()); + + // If MTCNN model not available, use HOG + if (!face_detector.IsMTCNNLoaded()) + { + model_params.SetFaceDetector(false, true, false); + } + CLNF face_model = new CLNF(model_params); GazeAnalyserManaged gaze_analyser = new GazeAnalyserManaged(); @@ -417,11 +428,12 @@ namespace HeadPoseLive List landmarks = new List(); List> gaze_lines = null; Tuple gaze_angle = new Tuple(0, 0); + var visibilities = face_model.GetVisibilities(); double scale = face_model.GetRigidParams()[0]; if (detectionSucceeding) { - List> landmarks_doubles = face_model.CalculateVisibleLandmarks(); + List> landmarks_doubles = face_model.CalculateAllLandmarks(); foreach (var p in landmarks_doubles) landmarks.Add(new System.Windows.Point(p.Item1, p.Item2)); @@ -525,6 +537,7 @@ namespace HeadPoseLive { webcam_img.OverlayLines.Add(lines); webcam_img.OverlayPoints.Add(landmarks); + webcam_img.OverlayPointsVisibility.Add(visibilities); webcam_img.FaceScale.Add(scale); List eye_landmark_points = new List(); diff --git a/gui/OpenFaceDemo/MainWindow.xaml.cs b/gui/OpenFaceDemo/MainWindow.xaml.cs index a10c3ae1..a897f33a 100644 --- a/gui/OpenFaceDemo/MainWindow.xaml.cs +++ b/gui/OpenFaceDemo/MainWindow.xaml.cs @@ -50,6 +50,7 @@ using OpenFaceOffline; using OpenCVWrappers; using CppInterop.LandmarkDetector; using FaceAnalyser_Interop; +using FaceDetectorInterop; using GazeAnalyser_Interop; using UtilitiesOF; @@ -119,6 +120,15 @@ namespace OpenFaceDemo face_model_params = new FaceModelParameters(root, true, false, false); face_model_params.optimiseForVideo(); + // Initialize the face detector + FaceDetector face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation()); + + // If MTCNN model not available, use HOG + if (!face_detector.IsMTCNNLoaded()) + { + face_model_params.SetFaceDetector(false, true, false); + } + landmark_detector = new CLNF(face_model_params); face_analyser = new FaceAnalyserManaged(root, true, 112, true); gaze_analyser = new GazeAnalyserManaged(); diff --git a/gui/OpenFaceOffline/MainWindow.xaml b/gui/OpenFaceOffline/MainWindow.xaml index 5ba9a2ec..05524894 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml +++ b/gui/OpenFaceOffline/MainWindow.xaml @@ -64,24 +64,15 @@ - - - - - - - - + + + + - - - - - - - - + + + diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index e4af3856..ad0e7220 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -156,8 +156,20 @@ namespace OpenFaceOffline this.Icon = BitmapFrame.Create(iconUri); String root = AppDomain.CurrentDomain.BaseDirectory; - + face_model_params = new FaceModelParameters(root, LandmarkDetectorCECLM, LandmarkDetectorCLNF, LandmarkDetectorCLM); + // Initialize the face detector + face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation()); + + // If MTCNN model not available, use HOG + if (!face_detector.IsMTCNNLoaded()) + { + FaceDetCNN.IsEnabled = false; + DetectorCNN = false; + DetectorHOG = true; + } + face_model_params.SetFaceDetector(DetectorHaar, DetectorHOG, DetectorCNN); + landmark_detector = new CLNF(face_model_params); gaze_analyser = new GazeAnalyserManaged(); @@ -980,6 +992,18 @@ namespace OpenFaceOffline } } + private void ExclusiveMenuItem_Click(object sender, RoutedEventArgs e) + { + // Disable all other items but this one + MenuItem parent = (MenuItem)((MenuItem)sender).Parent; + foreach (var me in parent.Items) + { + ((MenuItem)me).IsChecked = false; + } + ((MenuItem)sender).IsChecked = true; + + } + private void setCameraParameters_Click(object sender, RoutedEventArgs e) { CameraParametersEntry camera_params_entry_window = new CameraParametersEntry(fx, fy, cx, cy); diff --git a/lib/local/CppInerop/FaceDetectorInterop.h b/lib/local/CppInerop/FaceDetectorInterop.h index 535a360d..f44b5b27 100644 --- a/lib/local/CppInerop/FaceDetectorInterop.h +++ b/lib/local/CppInerop/FaceDetectorInterop.h @@ -124,6 +124,11 @@ namespace FaceDetectorInterop { } } + bool IsMTCNNLoaded() + { + return !face_detector_mtcnn->empty(); + } + // Face detection using MTCNN face detector void DetectFacesMTCNN(List^ o_regions, OpenCVWrappers::RawImage^ rgb_image, List^ o_confidences) {