Integrating OpenFace 1.0.0 changes into CE-CLM branch

lib/local/CppInerop/LandmarkDetectorInterop.h

@@ -272,6 +272,9 @@ namespace CppInterop {
 				clnf->Reset(x, y);
 			}
 
+			bool HasEyeModel() {
+				return clnf->eye_model;
+			}
 
 			double GetConfidence()
 			{

lib/local/CppInerop/VisualizerInterop.h

@@ -54,9 +54,9 @@ namespace UtilitiesOF {
 
 	public:
 
-		Visualizer(bool vis_track, bool vis_hog, bool vis_aligned)
+		Visualizer(bool vis_track, bool vis_hog, bool vis_aligned, bool vis_aus)
 		{
-			m_visualizer = new Utilities::Visualizer(vis_track, vis_hog, vis_aligned);
+			m_visualizer = new Utilities::Visualizer(vis_track, vis_hog, vis_aligned, vis_aus);
 		}
 
 		void SetObservationGaze(System::Tuple<float, float, float>^ gaze_direction0, System::Tuple<float, float, float>^ gaze_direction1,

lib/local/FaceAnalyser/src/FaceAnalyser.cpp

@@ -1224,25 +1224,28 @@ void FaceAnalyser::ReadRegressor(std::string fname, const vector<string>& au_nam
 {
 	ifstream regressor_stream(fname.c_str(), ios::in | ios::binary);
 
-	// First read the input type
-	int regressor_type;
-	regressor_stream.read((char*)&regressor_type, 4);
+	if (regressor_stream.is_open())
+	{
+		// First read the input type
+		int regressor_type;
+		regressor_stream.read((char*)&regressor_type, 4);
 
-	if(regressor_type == SVR_appearance_static_linear)
-	{
-		AU_SVR_static_appearance_lin_regressors.Read(regressor_stream, au_names);		
-	}
-	else if(regressor_type == SVR_appearance_dynamic_linear)
-	{
-		AU_SVR_dynamic_appearance_lin_regressors.Read(regressor_stream, au_names);		
-	}
-	else if(regressor_type == SVM_linear_stat)
-	{
-		AU_SVM_static_appearance_lin.Read(regressor_stream, au_names);		
-	}
-	else if(regressor_type == SVM_linear_dyn)
-	{
-		AU_SVM_dynamic_appearance_lin.Read(regressor_stream, au_names);		
+		if (regressor_type == SVR_appearance_static_linear)
+		{
+			AU_SVR_static_appearance_lin_regressors.Read(regressor_stream, au_names);
+		}
+		else if (regressor_type == SVR_appearance_dynamic_linear)
+		{
+			AU_SVR_dynamic_appearance_lin_regressors.Read(regressor_stream, au_names);
+		}
+		else if (regressor_type == SVM_linear_stat)
+		{
+			AU_SVM_static_appearance_lin.Read(regressor_stream, au_names);
+		}
+		else if (regressor_type == SVM_linear_dyn)
+		{
+			AU_SVM_dynamic_appearance_lin.Read(regressor_stream, au_names);
+		}
 	}
 }
 

lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp

@@ -149,7 +149,7 @@ FaceAnalyserParameters::FaceAnalyserParameters(vector<string> &arguments):root()
 	}
 	else
 	{
-		std::cout << "Could not find the AU detection model to load" << std::endl;
+		std::cout << "Could not find the face analysis module to load" << std::endl;
 	}
 }
 
@@ -182,7 +182,7 @@ void FaceAnalyserParameters::init()
 	}
 	else
 	{
-		std::cout << "Could not find the AU detection model to load" << std::endl;
+		std::cout << "Could not find the face analysis module to load" << std::endl;
 	}
 
 	orientation_bins = vector<cv::Vec3d>();
@@ -226,7 +226,7 @@ void FaceAnalyserParameters::OptimizeForVideos()
 	}
 	else
 	{
-		std::cout << "Could not find the AU detection model to load" << std::endl;
+		std::cout << "Could not find the face analysis module to load" << std::endl;
 	}
 
 }

lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h

@@ -114,6 +114,8 @@ struct FaceModelParameters
 
 	private:
 		void init();
+		void check_model_path(const std::string& root = "/");
+
 };
 
 }

lib/local/LandmarkDetector/model/clnf_multi_pie.txt

@@ -0,0 +1,5 @@
+PDM pdms/Multi-PIE_aligned_PDM_68.txt
+Triangulations tris_68.txt
+PatchesCCNF patch_experts/ccnf_patches_0.25_multi_pie.txt
+PatchesCCNF patch_experts/ccnf_patches_0.35_multi_pie.txt
+PatchesCCNF patch_experts/ccnf_patches_0.5_multi_pie.txt

lib/local/LandmarkDetector/model/main_clnf_multi_pie.txt

@@ -0,0 +1,3 @@
+LandmarkDetector clnf_multi_pie.txt
+FaceDetConversion haarAlign.txt
+DetectionValidator detection_validation/validator_cnn.txt

lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp

@@ -57,6 +57,8 @@ FaceModelParameters::FaceModelParameters()
 {
 	// initialise the default values
 	init();
+	check_model_path();
+
 }
 
 FaceModelParameters::FaceModelParameters(vector<string> &arguments)
@@ -260,6 +262,33 @@ FaceModelParameters::FaceModelParameters(vector<string> &arguments)
 	{
 		std::cout << "Could not find the MTCNN face detector location" << std::endl;
 	}
+	check_model_path(root.string());
+}
+
+void FaceModelParameters::check_model_path(const std::string& root)
+{
+	// Make sure model_location is valid
+	// First check working directory, then the executable's directory, then the config path set by the build process.
+	boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR);
+	boost::filesystem::path model_path = boost::filesystem::path(model_location);
+	boost::filesystem::path root_path = boost::filesystem::path(root);
+
+	if (boost::filesystem::exists(model_path))
+	{
+		model_location = model_path.string();
+	}
+	else if (boost::filesystem::exists(root_path / model_path))
+	{
+		model_location = (root_path / model_path).string();
+	}
+	else if (boost::filesystem::exists(config_path / model_path))
+	{
+		model_location = (config_path / model_path).string();
+	}
+	else
+	{
+		std::cout << "Could not find the landmark detection model to load" << std::endl;
+	}
 }
 
 void FaceModelParameters::init()

lib/local/Utilities/include/RecorderOpenFaceParameters.h

@@ -78,6 +78,7 @@ namespace Utilities
 		float getCy() const { return cy; }
 
 		void setOutputAUs(bool output_AUs) { this->output_AUs = output_AUs; }
+		void setOutputGaze(bool output_gaze) { this->output_gaze = output_gaze; }
 
 	private:
 		

lib/local/Utilities/include/Visualizer.h

@@ -54,8 +54,8 @@ namespace Utilities
 
 		// The constructor for the visualizer that specifies what to visualize
 		Visualizer(std::vector<std::string> arguments);
-		Visualizer(bool vis_track, bool vis_hog, bool vis_align);
-		
+		Visualizer(bool vis_track, bool vis_hog, bool vis_align, bool vis_aus);
+
 		// Adding observations to the visualizer
 		
 		// Pose related observations
@@ -67,6 +67,8 @@ namespace Utilities
 		// Pose related observations
 		void SetObservationPose(const cv::Vec6f& pose, double confidence);
 		
+		void SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);
+
 		// Gaze related observations
 		void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector<cv::Point2f>& eye_landmarks, const std::vector<cv::Point3f>& eye_landmarks3d, double confidence);
 
@@ -88,7 +90,8 @@ namespace Utilities
 		bool vis_track;
 		bool vis_hog;
 		bool vis_align;
-		
+		bool vis_aus;
+
 		// Can be adjusted to show less confident frames
 		double visualisation_boundary = 0.4;
 
@@ -99,6 +102,7 @@ namespace Utilities
 		cv::Mat tracked_image;
 		cv::Mat hog_image;
 		cv::Mat aligned_face_image;
+		cv::Mat action_units_image;
 
 		// Useful for drawing 3d
 		float fx, fy, cx, cy;

lib/local/Utilities/src/Visualizer.cpp

@@ -36,6 +36,11 @@
 #include "RotationHelpers.h"
 #include "ImageManipulationHelpers.h"
 
+#include <sstream>
+#include <iomanip>
+#include <map>
+#include <set>
+
 // For drawing on images
 #include <opencv2/imgproc.hpp>
 
@@ -45,12 +50,35 @@ using namespace Utilities;
 const int draw_shiftbits = 4;
 const int draw_multiplier = 1 << 4;
 
+const std::map<std::string, std::string> AUS_DESCRIPTION = {
+	{ "AU01", "Inner Brow Raiser   " },
+	{ "AU02", "Outer Brow Raiser   " },
+	{ "AU04", "Brow Lowerer        " },
+	{ "AU05", "Upper Lid Raiser    " },
+	{ "AU06", "Cheek Raiser        " },
+	{ "AU07", "Lid Tightener       " },
+	{ "AU09", "Nose Wrinkler       " },
+	{ "AU10", "Upper Lip Raiser    " },
+	{ "AU12", "Lip Corner Puller   " },
+	{ "AU14", "Dimpler             " },
+	{ "AU15", "Lip Corner Depressor" },
+	{ "AU17", "Chin Raiser         " },
+	{ "AU20", "Lip stretcher       " },
+	{ "AU23", "Lip Tightener       " },
+	{ "AU25", "Lips part           " },
+	{ "AU26", "Jaw Drop            " },
+	{ "AU28", "Lip Suck            " },
+	{ "AU45", "Blink               " },
+
+};
+
 Visualizer::Visualizer(std::vector<std::string> arguments)
 {
 	// By default not visualizing anything
 	this->vis_track = false;
 	this->vis_hog = false;
 	this->vis_align = false;
+	this->vis_aus = false;
 
 	for (size_t i = 0; i < arguments.size(); ++i)
 	{
@@ -59,6 +87,7 @@ Visualizer::Visualizer(std::vector<std::string> arguments)
 			vis_track = true;
 			vis_align = true;
 			vis_hog = true;
+			vis_aus = true;
 		}
 		else if (arguments[i].compare("-vis-align") == 0)
 		{
@@ -72,19 +101,22 @@ Visualizer::Visualizer(std::vector<std::string> arguments)
 		{
 			vis_track = true;
 		}
+		else if (arguments[i].compare("-vis-aus") == 0)
+		{
+			vis_aus = true;
+		}
 	}
 
 }
 
-Visualizer::Visualizer(bool vis_track, bool vis_hog, bool vis_align)
+Visualizer::Visualizer(bool vis_track, bool vis_hog, bool vis_align, bool vis_aus)
 {
 	this->vis_track = vis_track;
 	this->vis_hog = vis_hog;
 	this->vis_align = vis_align;
+	this->vis_aus = vis_aus;
 }
 
-
-
 // Setting the image on which to draw
 void Visualizer::SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy)
 {
@@ -99,6 +131,7 @@ void Visualizer::SetImage(const cv::Mat& canvas, float fx, float fy, float cx, f
 	// Clearing other images
 	hog_image = cv::Mat();
 	aligned_face_image = cv::Mat();
+	action_units_image = cv::Mat();
 
 }
 
@@ -195,6 +228,108 @@ void Visualizer::SetObservationPose(const cv::Vec6f& pose, double confidence)
 	}
 }
 
+void Visualizer::SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities,
+	const std::vector<std::pair<std::string, double> >& au_occurences)
+{
+	if (au_intensities.size() > 0 || au_occurences.size() > 0)
+	{
+
+		std::set<std::string> au_names;
+		std::map<std::string, bool> occurences_map;
+		std::map<std::string, double> intensities_map;
+
+		for (size_t idx = 0; idx < au_intensities.size(); idx++)
+		{
+			au_names.insert(au_intensities[idx].first);
+			intensities_map[au_intensities[idx].first] = au_intensities[idx].second;
+		}
+
+		for (size_t idx = 0; idx < au_occurences.size(); idx++)
+		{
+			au_names.insert(au_occurences[idx].first);
+			occurences_map[au_occurences[idx].first] = au_occurences[idx].second;
+		}
+
+		const int AU_TRACKBAR_LENGTH = 400;
+		const int AU_TRACKBAR_HEIGHT = 10;
+
+		const int MARGIN_X = 185;
+		const int MARGIN_Y = 10;
+
+		const int nb_aus = au_names.size();
+
+		// Do not reinitialize
+		if (action_units_image.empty())
+		{
+			action_units_image = cv::Mat(nb_aus * (AU_TRACKBAR_HEIGHT + 10) + MARGIN_Y * 2, AU_TRACKBAR_LENGTH + MARGIN_X, CV_8UC3, cv::Scalar(255, 255, 255));
+		}
+		else
+		{
+			action_units_image.setTo(255);
+		}
+
+		std::map<std::string, std::pair<bool, double>> aus;
+
+		// first, prepare a mapping "AU name" -> { present, intensity }
+		for (auto au_name : au_names)
+		{
+			// Insert the intensity and AU presense (as these do not always overlap check if they exist first)
+			bool occurence = false;
+			if (occurences_map.find(au_name) != occurences_map.end())
+			{
+				occurence = occurences_map[au_name] != 0;
+			}
+			else
+			{
+				// If we do not have an occurence label, trust the intensity one
+				occurence = intensities_map[au_name] > 1;
+			}
+			double intensity = 0.0;
+			if (intensities_map.find(au_name) != intensities_map.end())
+			{
+				intensity = intensities_map[au_name];
+			}
+			else
+			{
+				// If we do not have an intensity label, trust the occurence one
+				intensity = occurences_map[au_name] == 0 ? 0 : 5;
+			}
+
+			aus[au_name] = std::make_pair(occurence, intensity);
+		}
+
+		// then, build the graph
+		size_t idx = 0;
+		for (auto& au : aus)
+		{
+			std::string name = au.first;
+			bool present = au.second.first;
+			double intensity = au.second.second;
+
+			auto offset = MARGIN_Y + idx * (AU_TRACKBAR_HEIGHT + 10);
+			std::ostringstream au_i;
+			au_i << std::setprecision(2) << std::setw(4) << std::fixed << intensity;
+			cv::putText(action_units_image, name, cv::Point(10, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(present ? 0 : 200, 0, 0), 1, CV_AA);
+			cv::putText(action_units_image, AUS_DESCRIPTION.at(name), cv::Point(55, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 0, 0), 1, CV_AA);
+
+			if (present)
+			{
+				cv::putText(action_units_image, au_i.str(), cv::Point(160, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 100, 0), 1, CV_AA);
+				cv::rectangle(action_units_image, cv::Point(MARGIN_X, offset),
+					cv::Point(MARGIN_X + AU_TRACKBAR_LENGTH * intensity / 5, offset + AU_TRACKBAR_HEIGHT),
+					cv::Scalar(128, 128, 128),
+					CV_FILLED);
+			}
+			else
+			{
+				cv::putText(action_units_image, "0.00", cv::Point(160, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 0, 0), 1, CV_AA);
+			}
+			idx++;
+		}
+	}
+}
+
+
 // Eye gaze infomration drawing, first of eye landmarks then of gaze
 void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector<cv::Point2f>& eye_landmarks2d, const std::vector<cv::Point3f>& eye_landmarks3d, double confidence)
 {
@@ -283,19 +418,22 @@ void Visualizer::SetFps(double fps)
 
 char Visualizer::ShowObservation()
 {
-	if (vis_track)
-	{
-		cv::namedWindow("tracking_result", 1);
-		cv::imshow("tracking_result", captured_image);
-	}
-	if (vis_align)
+	if (vis_align && !aligned_face_image.empty())
 	{
 		cv::imshow("sim_warp", aligned_face_image);
 	}
-	if (vis_hog)
+	if (vis_hog && !hog_image.empty())
 	{
 		cv::imshow("hog", hog_image);
 	}
+	if (vis_aus && !action_units_image.empty())
+	{
+		cv::imshow("action units", action_units_image);
+	}
+	if (vis_track)
+	{
+		cv::imshow("tracking result", captured_image);
+	}
 	return cv::waitKey(1);
 }