diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index d6075aef..b36238fe 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -49,6 +49,7 @@ using CppInterop.LandmarkDetector; using CameraInterop; using FaceAnalyser_Interop; using GazeAnalyser_Interop; +using MediaReader; using System.Globalization; using Microsoft.WindowsAPICodePack.Dialogs; @@ -85,6 +86,7 @@ namespace OpenFaceOffline // Some members for displaying the results private Capture capture; + private ImageReader image_reader; private WriteableBitmap latest_img; private WriteableBitmap latest_aligned_face; private WriteableBitmap latest_HOG_descriptor; diff --git a/lib/local/CppInerop/CppInerop.vcxproj b/lib/local/CppInerop/CppInerop.vcxproj index acc08c09..df571101 100644 --- a/lib/local/CppInerop/CppInerop.vcxproj +++ b/lib/local/CppInerop/CppInerop.vcxproj @@ -116,7 +116,7 @@ Level3 Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions) - ./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + ./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) Windows @@ -130,7 +130,7 @@ Level3 Disabled _DEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions) - ./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + ./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) Windows @@ -146,7 +146,7 @@ true true WIN32;NDEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions) - ./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + ./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) Windows @@ -164,7 +164,7 @@ true true NDEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions) - ./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + ./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) Windows @@ -181,6 +181,7 @@ + @@ -200,6 +201,9 @@ {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} + + {8e741ea2-9386-4cf2-815e-6f9b08991eac} + diff --git a/lib/local/CppInerop/CppInerop.vcxproj.filters b/lib/local/CppInerop/CppInerop.vcxproj.filters index abe126c3..f830fbc1 100644 --- a/lib/local/CppInerop/CppInerop.vcxproj.filters +++ b/lib/local/CppInerop/CppInerop.vcxproj.filters @@ -38,5 +38,8 @@ Header Files + + Header Files + \ No newline at end of file diff --git a/lib/local/CppInerop/CppInterop.cpp b/lib/local/CppInerop/CppInterop.cpp index d7804f6c..a0cc5a43 100644 --- a/lib/local/CppInerop/CppInterop.cpp +++ b/lib/local/CppInerop/CppInterop.cpp @@ -38,3 +38,4 @@ #include "GazeAnalyserInterop.h" #include "OpenCVWrappers.h" #include "CameraInterop.h" +#include "ImageReader.h" diff --git a/lib/local/CppInerop/FaceAnalyserInterop.h b/lib/local/CppInerop/FaceAnalyserInterop.h index 5603f62f..74d54336 100644 --- a/lib/local/CppInerop/FaceAnalyserInterop.h +++ b/lib/local/CppInerop/FaceAnalyserInterop.h @@ -61,6 +61,7 @@ #include #include #include +#include // Boost stuff #include @@ -187,7 +188,7 @@ public: face_analyser->PostprocessOutputFile(msclr::interop::marshal_as(file)); } - void AddNextFrame(OpenCVWrappers::RawImage^ frame, List^>^ landmarks, bool success, bool online) { + void AddNextFrame(OpenCVWrappers::RawImage^ frame, List^>^ landmarks, bool success, bool online, bool vis_hog) { // Construct an OpenCV matric from the landmarks cv::Mat_ landmarks_mat(landmarks->Count * 2, 1, 0.0); @@ -209,7 +210,7 @@ public: if(vis_hog) { - *visualisation = face_analyser->GetLatestHOGDescriptorVisualisation(); + Utilities::Visualise_FHOG(*hog_features, *num_rows, *num_cols, *visualisation); } } @@ -227,10 +228,10 @@ public: landmarks_mat.at(i + landmarks->Count, 0) = landmarks[i]->Item2; } - auto aus = face_analyser->PredictStaticAUs(frame->Mat, landmarks_mat, false); + face_analyser->PredictStaticAUsAndComputeFeatures(frame->Mat, landmarks_mat); - auto AU_predictions_intensity = aus.first; - auto AU_predictions_occurence = aus.second; + auto AU_predictions_intensity = face_analyser->GetCurrentAUsReg(); + auto AU_predictions_occurence = face_analyser->GetCurrentAUsClass(); auto au_intensities = gcnew Dictionary(); auto au_occurences = gcnew Dictionary(); diff --git a/lib/local/CppInerop/ImageReader.h b/lib/local/CppInerop/ImageReader.h new file mode 100644 index 00000000..c8da5449 --- /dev/null +++ b/lib/local/CppInerop/ImageReader.h @@ -0,0 +1,191 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt + +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +// Camera_Interop.h + +#pragma once + +#pragma unmanaged + +// Include all the unmanaged things we need. + +#include +#include "opencv2/objdetect.hpp" +#include "opencv2/calib3d.hpp" +#include +#include +#include +#include +#include + +#include + +#include "ImageCapture.h" + +#pragma managed + +#include +#include + +namespace MediaReader { + + public ref class ReadingFailedException : System::Exception + { + public: + + ReadingFailedException(System::String^ message) : Exception(message) {} + }; + + public ref class ImageReader + { + private: + + // OpenCV based video capture for reading from files + Utilities::ImageCapture* m_image_capture; + + OpenCVWrappers::RawImage^ m_rgb_frame; + OpenCVWrappers::RawImage^ m_gray_frame; + + bool* m_is_opened; + + public: + + // Can provide a directory, or a list of files + ImageReader(System::String^ image_directory) + { + m_image_capture = new Utilities::ImageCapture(); + + std::string image_dir_std = msclr::interop::marshal_as(image_directory); + + *m_is_opened = m_image_capture->OpenDirectory(image_dir_std); + + if (*m_is_opened) + { + throw gcnew ReadingFailedException("Failed to open a directory or an image"); + } + } + // Can provide a directory, or a list of files + ImageReader(System::Collections::Generic::List^ image_files) + { + m_image_capture = new Utilities::ImageCapture(); + + std::vector image_files_std; + + for (size_t i = 0; i < image_files->Count; ++i) + { + std::string image_file = msclr::interop::marshal_as(image_files[i]); + image_files_std.push_back(image_file); + } + + bool success = m_image_capture->OpenImageFiles(image_files_std); + + if (!success) + { + throw gcnew ReadingFailedException("Failed to open a directory or an image"); + } + + } + + OpenCVWrappers::RawImage^ GetNextImage() + { + cv::Mat next_image = m_image_capture->GetNextImage(); + + if (m_rgb_frame == nullptr) + { + m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().width, CV_8UC3); + } + + next_image.copyTo(m_rgb_frame->Mat); + + return m_rgb_frame; + } + + double GetProgress() + { + return m_image_capture->GetProgress(); + } + + bool isOpened() + { + return *m_is_opened; + } + + OpenCVWrappers::RawImage^ GetCurrentFrameGray() { + + cv::Mat next_gray_image = m_image_capture->GetGrayFrame(); + + if (m_gray_frame == nullptr) + { + m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().width, CV_8UC3); + } + + next_gray_image.copyTo(m_gray_frame->Mat); + + return m_gray_frame; + } + + // Finalizer. Definitely called before Garbage Collection, + // but not automatically called on explicit Dispose(). + // May be called multiple times. + !ImageReader() + { + // Automatically closes capture object before freeing memory. + if (m_image_capture != nullptr) + { + delete m_image_capture; + } + + if (m_rgb_frame != nullptr) + { + delete m_rgb_frame; + } + if (m_gray_frame != nullptr) + { + delete m_gray_frame; + } + if (m_is_opened != nullptr) + { + delete m_gray_frame; + } + + } + + // Destructor. Called on explicit Dispose() only. + ~ImageReader() + { + this->!ImageReader(); + } + }; + +} diff --git a/lib/local/CppInerop/LandmarkDetectorInterop.h b/lib/local/CppInerop/LandmarkDetectorInterop.h index b69982bc..20fbcc35 100644 --- a/lib/local/CppInerop/LandmarkDetectorInterop.h +++ b/lib/local/CppInerop/LandmarkDetectorInterop.h @@ -63,6 +63,7 @@ #include #include +#include #ifdef __cplusplus_cli #undef generic @@ -325,26 +326,11 @@ namespace CppInterop { return landmarks_3D; } - - // Static functions from the LandmarkDetector namespace. - void DrawLandmarks(OpenCVWrappers::RawImage^ img, List^ landmarks) { - - vector vecLandmarks; - - for(int i = 0; i < landmarks->Count; i++) { - System::Windows::Point p = landmarks[i]; - vecLandmarks.push_back(cv::Point(p.X, p.Y)); - } - - ::LandmarkDetector::DrawLandmarks(img->Mat, vecLandmarks); - } - - List^>^ CalculateBox(float fx, float fy, float cx, float cy) { cv::Vec6d pose = ::LandmarkDetector::GetPose(*clnf, fx,fy, cx, cy); - vector> vecLines = ::LandmarkDetector::CalculateBox(pose, fx, fy, cx, cy); + vector> vecLines = ::Utilities::CalculateBox(pose, fx, fy, cx, cy); auto lines = gcnew List^>(); @@ -355,19 +341,6 @@ namespace CppInterop { return lines; } - void DrawBox(List^>^ lines, OpenCVWrappers::RawImage^ image, double r, double g, double b, int thickness) { - cv::Scalar color = cv::Scalar(r,g,b,1); - - vector> vecLines; - - for(int i = 0; i < lines->Count; i++) { - System::Tuple^ points = lines[i]; - vecLines.push_back(pair(cv::Point(points->Item1.X, points->Item1.Y), cv::Point(points->Item2.X, points->Item2.Y))); - } - - ::LandmarkDetector::DrawBox(vecLines, image->Mat, color, thickness); - } - int GetNumPoints() { return clnf->pdm.NumberOfPoints(); diff --git a/lib/local/Utilities/include/VisualizationUtils.h b/lib/local/Utilities/include/VisualizationUtils.h index 518610f0..b04348b4 100644 --- a/lib/local/Utilities/include/VisualizationUtils.h +++ b/lib/local/Utilities/include/VisualizationUtils.h @@ -51,7 +51,7 @@ namespace Utilities // Computing a bounding box to be drawn std::vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); - void Visualise_FHOG(const cv::Mat_& descriptor, int num_rows, int num_cols, cv::Mat& visualisation); + void Visualise_FHOG(const cv::Mat_& descriptor, int num_rows, int num_cols, cv::Mat& visualisation); class FpsTracker {