diff --git a/gui/OpenFaceDemo/MainWindow.xaml.cs b/gui/OpenFaceDemo/MainWindow.xaml.cs index 61d51a6c..7bd25c74 100644 --- a/gui/OpenFaceDemo/MainWindow.xaml.cs +++ b/gui/OpenFaceDemo/MainWindow.xaml.cs @@ -116,7 +116,7 @@ namespace OpenFaceDemo String root = AppDomain.CurrentDomain.BaseDirectory; // TODO, create a demo version of parameters - face_model_params = new FaceModelParameters(root, false); + face_model_params = new FaceModelParameters(root, true, false, false); face_model_params.optimiseForVideo(); landmark_detector = new CLNF(face_model_params); @@ -246,7 +246,8 @@ namespace OpenFaceDemo List> landmarks = null; List> eye_landmarks = null; List> gaze_lines = null; - + List visibilities = null; + Tuple gaze_angle = gaze_analyser.GetGazeAngle(); if (detection_succeeding) @@ -255,6 +256,7 @@ namespace OpenFaceDemo eye_landmarks = landmark_detector.CalculateVisibleEyeLandmarks(); lines = landmark_detector.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); + visibilities = landmark_detector.GetVisibilities(); } // Visualisation @@ -344,9 +346,11 @@ namespace OpenFaceDemo eye_landmark_points.Add(new Point(p.Item1, p.Item2)); } - video.OverlayPoints.Add(landmark_points); video.OverlayEyePoints.Add(eye_landmark_points); + video.OverlayPointsVisibility.Add(visibilities); + video.OverlayEyePoints.Add(eye_landmark_points); + video.GazeLines.Add(gaze_lines); } diff --git a/gui/OpenFaceDemo/OpenFaceDemo.csproj b/gui/OpenFaceDemo/OpenFaceDemo.csproj index 204e1f47..c653255a 100644 --- a/gui/OpenFaceDemo/OpenFaceDemo.csproj +++ b/gui/OpenFaceDemo/OpenFaceDemo.csproj @@ -33,7 +33,7 @@ pdbonly true ..\..\x64\Release\ - TRACE + DEBUG;TRACE prompt 4 diff --git a/gui/OpenFaceOffline/MainWindow.xaml b/gui/OpenFaceOffline/MainWindow.xaml index f54d96e0..0e56cb0d 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml +++ b/gui/OpenFaceOffline/MainWindow.xaml @@ -75,9 +75,9 @@ - - - + + + diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index 9e2582a0..c0299a10 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -101,7 +101,7 @@ namespace OpenFaceOffline // For tracking FaceDetector face_detector; - FaceModelParameters face_model_params; + FaceModelParameters face_model_params; // TODO does this need to be reinitialized every time to deal with model reloading? CLNF landmark_detector; // For face analysis @@ -135,6 +135,11 @@ namespace OpenFaceOffline public bool DetectorHOG { get; set; } = false; public bool DetectorCNN { get; set; } = true; + // Selecting which landmark detector will be used + public bool LandmarkDetectorCLM { get; set; } = false; + public bool LandmarkDetectorCLNF { get; set; } = false; + public bool LandmarkDetectorCECLM { get; set; } = true; + // For AU prediction, if videos are long dynamic models should be used public bool DynamicAUModels { get; set; } = true; @@ -150,12 +155,9 @@ namespace OpenFaceOffline Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute); this.Icon = BitmapFrame.Create(iconUri); - // Initialize the default face detectors and landmark detectors - - //as MenuItemWithRadioButton; String root = AppDomain.CurrentDomain.BaseDirectory; - face_model_params = new FaceModelParameters(root, false); + face_model_params = new FaceModelParameters(root, LandmarkDetectorCECLM, LandmarkDetectorCLNF, LandmarkDetectorCLM); landmark_detector = new CLNF(face_model_params); gaze_analyser = new GazeAnalyserManaged(); @@ -192,6 +194,11 @@ namespace OpenFaceOffline thread_running = true; + // Reload the face landmark detector if needed + ReloadLandmarkDetector(); + + // Set the face detector + face_model_params.SetFaceDetector(DetectorHaar, DetectorHOG, DetectorCNN); face_model_params.optimiseForVideo(); // Setup the visualization @@ -220,8 +227,9 @@ namespace OpenFaceOffline var lastFrameTime = CurrentTime; // Empty image would indicate that the stream is over - while (gray_frame.Width != 0) + while (!gray_frame.IsEmpty) { + if(!thread_running) { break; @@ -232,14 +240,14 @@ namespace OpenFaceOffline bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(frame, face_model_params, gray_frame); // The face analysis step (for AUs and eye gaze) - //face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, false); - //gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); + face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, false); + gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Only the final face will contain the details - //VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), landmark_detector.GetVisibilities(), detection_succeeding, true, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); + VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), landmark_detector.GetVisibilities(), detection_succeeding, true, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); // Record an observation - //RecordObservation(recorder, visualizer_of.GetVisImage(), 0, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetTimestamp(), reader.GetFrameNumber()); + RecordObservation(recorder, visualizer_of.GetVisImage(), 0, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetTimestamp(), reader.GetFrameNumber()); while (thread_running & thread_paused && skip_frames == 0) { @@ -280,16 +288,19 @@ namespace OpenFaceOffline // Indicate we will start running the thread thread_running = true; + // Reload the face landmark detector if needed + ReloadLandmarkDetector(); // Setup the parameters optimized for working on individual images rather than sequences face_model_params.optimiseForImages(); + // Setup the visualization Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance); // Initialize the face detector if it has not been initialized yet if (face_detector == null) { - face_detector = new FaceDetector(face_model_params.GetMTCNNLocation()); + face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation()); } // Initialize the face analyser @@ -330,6 +341,10 @@ namespace OpenFaceOffline { face_detector.DetectFacesMTCNN(face_detections, frame, confidences); } + else if(DetectorHaar) + { + face_detector.DetectFacesHaar(face_detections, gray_frame, confidences); + } // For visualization double progress = reader.GetProgress(); @@ -372,6 +387,34 @@ namespace OpenFaceOffline } + // If the landmark detector model changed need to reload it + private void ReloadLandmarkDetector() + { + bool reload = false; + if (face_model_params.IsCECLM() && !LandmarkDetectorCECLM) + { + reload = true; + } + else if(face_model_params.IsCLNF() && !LandmarkDetectorCLNF) + { + reload = true; + } + else if (face_model_params.IsCLM() && !LandmarkDetectorCLM) + { + reload = true; + } + + if(reload) + { + String root = AppDomain.CurrentDomain.BaseDirectory; + + face_model_params = new FaceModelParameters(root, LandmarkDetectorCECLM, LandmarkDetectorCLNF, LandmarkDetectorCLM); + landmark_detector = new CLNF(face_model_params); + } + } + + + private void RecordObservation(RecorderOpenFace recorder, RawImage vis_image, int face_id, bool success, float fx, float fy, float cx, float cy, double timestamp, int frame_number) { @@ -738,11 +781,14 @@ namespace OpenFaceOffline StopTracking(); var image_files = openMediaDialog(true); - ImageReader reader = new ImageReader(image_files, fx, fy, cx, cy); - processing_thread = new Thread(() => ProcessIndividualImages(reader)); - processing_thread.Start(); + if(image_files.Count > 0) + { + ImageReader reader = new ImageReader(image_files, fx, fy, cx, cy); + processing_thread = new Thread(() => ProcessIndividualImages(reader)); + processing_thread.Start(); + } } // Selecting a directory containing images diff --git a/lib/local/CppInerop/FaceAnalyserInterop.h b/lib/local/CppInerop/FaceAnalyserInterop.h index ca7a0f51..7843cfc2 100644 --- a/lib/local/CppInerop/FaceAnalyserInterop.h +++ b/lib/local/CppInerop/FaceAnalyserInterop.h @@ -275,6 +275,7 @@ public: // May be called multiple times. !FaceAnalyserManaged() { + // TODO check for null pointers delete hog_features; delete aligned_face; delete num_cols; diff --git a/lib/local/CppInerop/FaceDetectorInterop.h b/lib/local/CppInerop/FaceDetectorInterop.h index 50d7e1e5..535a360d 100644 --- a/lib/local/CppInerop/FaceDetectorInterop.h +++ b/lib/local/CppInerop/FaceDetectorInterop.h @@ -98,12 +98,31 @@ namespace FaceDetectorInterop { o_regions->Clear(); o_confidences->Clear(); - for(size_t i = 0; i < regions_ocv.size(); ++i) + for (size_t i = 0; i < regions_ocv.size(); ++i) { o_regions->Add(System::Windows::Rect(regions_ocv[i].x, regions_ocv[i].y, regions_ocv[i].width, regions_ocv[i].height)); o_confidences->Add(confidences_std[i]); } } + + // Face detection using HOG-SVM classifier + void DetectFacesHaar(List^ o_regions, OpenCVWrappers::RawImage^ intensity, List^ o_confidences) + { + + std::vector > regions_ocv; + + ::LandmarkDetector::DetectFaces(regions_ocv, intensity->Mat, *face_detector_haar); + + o_regions->Clear(); + o_confidences->Clear(); + + for(size_t i = 0; i < regions_ocv.size(); ++i) + { + o_regions->Add(System::Windows::Rect(regions_ocv[i].x, regions_ocv[i].y, regions_ocv[i].width, regions_ocv[i].height)); + // As Haar does not provide confidence, create a fake value + o_confidences->Add(1); + } + } // Face detection using MTCNN face detector void DetectFacesMTCNN(List^ o_regions, OpenCVWrappers::RawImage^ rgb_image, List^ o_confidences) @@ -128,10 +147,18 @@ namespace FaceDetectorInterop { // May be called multiple times. !FaceDetector() { - // TODO need only basis (might be null) - delete face_detector_hog; - delete face_detector_mtcnn; - delete face_detector_haar; + if (face_detector_hog != nullptr) + { + delete face_detector_hog; + } + if (face_detector_mtcnn != nullptr) + { + delete face_detector_mtcnn; + } + if (face_detector_haar != nullptr) + { + delete face_detector_haar; + } } // Destructor. Called on explicit Dispose() only. diff --git a/lib/local/CppInerop/ImageReader.h b/lib/local/CppInerop/ImageReader.h index afba502e..da3f322c 100644 --- a/lib/local/CppInerop/ImageReader.h +++ b/lib/local/CppInerop/ImageReader.h @@ -123,7 +123,7 @@ namespace UtilitiesOF { if (m_rgb_frame == nullptr) { - m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().width, CV_8UC3); + m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().height, CV_8UC3); } next_image.copyTo(m_rgb_frame->Mat); @@ -178,7 +178,7 @@ namespace UtilitiesOF { if (m_gray_frame == nullptr) { - m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().width, CV_8UC3); + m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().height, CV_8UC1); } next_gray_image.copyTo(m_gray_frame->Mat); diff --git a/lib/local/CppInerop/LandmarkDetectorInterop.h b/lib/local/CppInerop/LandmarkDetectorInterop.h index db3feffc..c64775a1 100644 --- a/lib/local/CppInerop/LandmarkDetectorInterop.h +++ b/lib/local/CppInerop/LandmarkDetectorInterop.h @@ -85,18 +85,32 @@ namespace CppInterop { public: // Initialise the parameters - FaceModelParameters(System::String^ root, bool demo) + FaceModelParameters(System::String^ root, bool ceclm, bool clnf, bool clm) { std::string root_std = msclr::interop::marshal_as(root); vector args; args.push_back(root_std); + std::string model_loc = "model/main_ceclm_general.txt"; + if (ceclm) + { + model_loc = "model/main_ceclm_general.txt"; + } + else if(clnf) + { + model_loc = "model/main_clnf_general.txt"; + } + else if (clm) + { + model_loc = "model/main_clm_general.txt"; + + } + + args.push_back("-mloc"); + args.push_back(model_loc); + params = new ::LandmarkDetector::FaceModelParameters(args); - if(demo) - { - params->model_location = "model/main_clnf_demos.txt"; - } } @@ -129,18 +143,54 @@ namespace CppInterop { params->weight_factor = 0; // Parameter optimizations for CE-CLM - if (params->is_ceclm_model) + if (params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::CECLM_DETECTOR) { params->sigma = 1.5f * params->sigma; params->reg_factor = 0.9f * params->reg_factor; } } + bool IsCECLM() + { + return params->curr_face_detector == ::LandmarkDetector::FaceModelParameters::CECLM_DETECTOR; + } + + bool IsCLNF() + { + return params->curr_face_detector == ::LandmarkDetector::FaceModelParameters::CLNF_DETECTOR; + } + + bool IsCLM() + { + return params->curr_face_detector == ::LandmarkDetector::FaceModelParameters::CLM_DETECTOR; + } + System::String^ GetMTCNNLocation() { return gcnew System::String(params->mtcnn_face_detector_location.c_str()); } + System::String^ GetHaarLocation() + { + return gcnew System::String(params->haar_face_detector_location.c_str()); + } + + void SetFaceDetector(bool haar, bool hog, bool cnn) + { + if (cnn) + { + params->curr_face_detector = params->MTCNN_DETECTOR; + } + else if (hog) + { + params->curr_face_detector = params->HOG_SVM_DETECTOR; + } + else if (haar) + { + params->curr_face_detector = params->HAAR_DETECTOR; + } + } + void optimiseForImages() { params->window_sizes_init = vector(4); @@ -157,7 +207,7 @@ namespace CppInterop { params->num_optimisation_iteration = 10; // Parameter optimizations for CE-CLM - if (params->is_ceclm_model) + if (params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::MTCNN_DETECTOR) { params->sigma = 1.5f * params->sigma; params->reg_factor = 0.9f * params->reg_factor; diff --git a/lib/local/CppInerop/SequenceReader.h b/lib/local/CppInerop/SequenceReader.h index 01b74598..f6c88cb1 100644 --- a/lib/local/CppInerop/SequenceReader.h +++ b/lib/local/CppInerop/SequenceReader.h @@ -151,7 +151,7 @@ namespace UtilitiesOF { if (m_rgb_frame == nullptr) { - m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().width, CV_8UC3); + m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().height, CV_8UC3); } next_image.copyTo(m_rgb_frame->Mat); @@ -221,7 +221,7 @@ namespace UtilitiesOF { if (m_gray_frame == nullptr) { - m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().width, CV_8UC3); + m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().height, CV_8UC1); } next_gray_image.copyTo(m_gray_frame->Mat); diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h b/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h index f6f9d363..c6a5465d 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h @@ -84,7 +84,8 @@ struct FaceModelParameters bool multi_view; // Based on model location, this affects the parameter settings - bool is_ceclm_model; + enum LandmarkDetector { CLM_DETECTOR, CLNF_DETECTOR, CECLM_DETECTOR }; + LandmarkDetector curr_landmark_detector; // How often should face detection be used to attempt reinitialisation, every n frames (set to negative not to reinit) int reinit_video_every; diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp index 20580304..68a07ebf 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp @@ -268,14 +268,14 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &rgb_image, CLNF& cl { cv::Rect_ bounding_box; - - // If the face detector has not been initialised read it in - if(clnf_model.face_detector_HAAR.empty()) + + // If the face detector has not been initialised and we're using it, then read it in + if(clnf_model.face_detector_HAAR.empty() && params.curr_face_detector == params.HAAR_DETECTOR) { clnf_model.face_detector_HAAR.load(params.haar_face_detector_location); clnf_model.haar_face_detector_location = params.haar_face_detector_location; } - if (clnf_model.face_detector_MTCNN.empty()) + if (clnf_model.face_detector_MTCNN.empty() && params.curr_face_detector == params.MTCNN_DETECTOR) { clnf_model.face_detector_MTCNN.Read(params.mtcnn_face_detector_location); clnf_model.mtcnn_face_detector_location = params.haar_face_detector_location; @@ -405,6 +405,7 @@ bool DetectLandmarksInImageMultiHypBasic(const cv::Mat_ &grayscale_image, // Store the current best estimate float best_likelihood; + float best_detection_certainty; cv::Vec6f best_global_parameters; cv::Mat_ best_local_parameters; cv::Mat_ best_detected_landmarks; @@ -440,8 +441,9 @@ bool DetectLandmarksInImageMultiHypBasic(const cv::Mat_ &grayscale_image, best_local_parameters = clnf_model.params_local.clone(); best_detected_landmarks = clnf_model.detected_landmarks.clone(); best_landmark_likelihoods = clnf_model.landmark_likelihoods.clone(); + best_detection_certainty = clnf_model.detection_certainty; best_success = success; - + for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part) { best_likelihood_h[part] = clnf_model.hierarchical_models[part].model_likelihood; @@ -461,6 +463,7 @@ bool DetectLandmarksInImageMultiHypBasic(const cv::Mat_ &grayscale_image, clnf_model.detected_landmarks = best_detected_landmarks.clone(); clnf_model.detection_success = best_success; clnf_model.landmark_likelihoods = best_landmark_likelihoods.clone(); + clnf_model.detection_certainty = best_detection_certainty; for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part) { diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp index 6354c7a8..cbbefada 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp @@ -208,13 +208,17 @@ FaceModelParameters::FaceModelParameters(vector &arguments) if (model_path.stem().string().compare("main_ceclm_general") == 0) { - is_ceclm_model = true; + curr_landmark_detector = CECLM_DETECTOR; sigma = 1.5f * sigma; reg_factor = 0.9f * reg_factor; } - else + else if (model_path.stem().string().compare("main_clnf_general") == 0) { - is_ceclm_model = false; + curr_landmark_detector = CLNF_DETECTOR; + } + else if (model_path.stem().string().compare("main_clm_general") == 0) + { + curr_landmark_detector = CLM_DETECTOR; } // Make sure face detector location is valid @@ -296,7 +300,7 @@ void FaceModelParameters::init() window_sizes_current = window_sizes_init; model_location = "model/main_ceclm_general.txt"; - is_ceclm_model = true; + curr_landmark_detector = CECLM_DETECTOR; sigma = 1.5f; reg_factor = 25.0f; diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 6875234a..e316fa16 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -683,7 +683,7 @@ namespace LandmarkDetector } // Convert from int bounding box do a double one with corrections - for (size_t face = 0; face < o_regions.size(); ++face) + for (size_t face = 0; face < face_detections.size(); ++face) { // OpenCV is overgenerous with face size and y location is off // CLNF detector expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate @@ -798,9 +798,6 @@ namespace LandmarkDetector detector(cv_grayscale, face_detections, -0.2); // Convert from int bounding box do a double one with corrections - //o_regions.resize(face_detections.size()); - //o_confidences.resize(face_detections.size()); - for (size_t face = 0; face < face_detections.size(); ++face) { // CLNF expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate