Merge branch 'develop' into OpenFaceC

.travis.yml

@@ -13,7 +13,8 @@ compiler:
 
 os:
   - linux  
-  
+  - osx
+
 before_install:
 
   # OpenCV dependencies and boost
@@ -58,7 +59,6 @@ before_install:
       brew install tbb;
       brew tap homebrew/science;
       brew install opencv3;
-      brew install openblas;
       brew upgrade boost;
     fi
   
@@ -78,7 +78,9 @@ script:
   - mkdir build
   - cd build
   - cmake -D CMAKE_BUILD_TYPE=RELEASE CMAKE_CXX_FLAGS="-std=c++11" -D CMAKE_EXE_LINKER_FLAGS="-std=c++11" .. 
-  - make
-  - ../build/bin/FaceLandmarkImg -inroot ../samples -f sample1.jpg -out_dir data -of sample1.txt -multi_view 1 -wild -q
+  - make -j4
+  - export OMP_NUM_THREADS=1
+  - export VECLIB_MAXIMUM_THREADS=1
+  - ../build/bin/FaceLandmarkImg -f ../samples/sample1.jpg -f ../samples/sample3.jpg -out_dir data -multi_view 1 -wild -q
   - ../build/bin/FeatureExtraction -fdir "../samples/image_sequence" -out_dir output -q
-  - ../build/bin/FaceLandmarkVidMulti -fdir ../samples/image_sequence -q -mloc model/main_clnf_general.txt
+  - ../build/bin/FaceLandmarkVidMulti -fdir ../samples/image_sequence -q -mloc model/main_clm_general.txt

CMakeLists.txt

@@ -11,7 +11,7 @@ add_definitions(-DCONFIG_DIR="${CONFIG_DIR}")
 
 find_package( BLAS REQUIRED )
 include_directories( ${BLAS_INCLUDE_DIRS} )
-LINK_DIRECTORIES(${BLAS_LIBRARY_DIRS})	
+LINK_DIRECTORIES(${BLAS_LIBRARY_DIRS})
 
 find_package( OpenCV 3.3 REQUIRED )
 

cmake/modules/FindBLAS.cmake

@@ -1,6 +1,3 @@
-# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
-# file Copyright.txt or https://cmake.org/licensing for details.
-
 #.rst:
 # FindBLAS
 # --------
@@ -32,32 +29,26 @@
 #      all the possibilities
 #   BLA_F95     if set on tries to find the f95 interfaces for BLAS/LAPACK
 #
-# List of vendors (BLA_VENDOR) valid in this module:
+# ######### ## List of vendors (BLA_VENDOR) valid in this module #
+# Goto,OpenBLAS,ATLAS PhiPACK,CXML,DXML,SunPerf,SCSL,SGIMATH,IBMESSL,
+# Intel10_32 (intel mkl v10 32 bit),Intel10_64lp (intel mkl v10 64 bit,
+# lp thread model, lp64 model), # Intel10_64lp_seq (intel mkl v10 64
+# bit,sequential code, lp64 model), # Intel( older versions of mkl 32
+# and 64 bit), ACML,ACML_MP,ACML_GPU,Apple, NAS, Generic C/CXX should be
+# enabled to use Intel mkl
+
+#=============================================================================
+# Copyright 2007-2009 Kitware, Inc.
 #
-# * Goto
-# * OpenBLAS
-# * ATLAS PhiPACK
-# * CXML
-# * DXML
-# * SunPerf
-# * SCSL
-# * SGIMATH
-# * IBMESSL
-# * Intel10_32 (intel mkl v10 32 bit)
-# * Intel10_64lp (intel mkl v10 64 bit, lp thread model, lp64 model)
-# * Intel10_64lp_seq (intel mkl v10 64 bit, sequential code, lp64 model)
-# * Intel (older versions of mkl 32 and 64 bit)
-# * ACML
-# * ACML_MP
-# * ACML_GPU
-# * Apple
-# * NAS
-# * Generic
-#
-# .. note::
-#
-#   C/CXX should be enabled to use Intel mkl
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
 #
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
 
 include(${CMAKE_CURRENT_LIST_DIR}/CheckFunctionExists.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/CheckFortranFunctionExists.cmake)
@@ -68,7 +59,12 @@ set(CMAKE_REQUIRED_QUIET ${BLAS_FIND_QUIETLY})
 set(_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
 
 # Check the language being used
-if( NOT (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED OR CMAKE_Fortran_COMPILER_LOADED) )
+get_property( _LANGUAGES_ GLOBAL PROPERTY ENABLED_LANGUAGES )
+if( _LANGUAGES_ MATCHES Fortran )
+  set( _CHECK_FORTRAN TRUE )
+elseif( (_LANGUAGES_ MATCHES C) OR (_LANGUAGES_ MATCHES CXX) )
+  set( _CHECK_FORTRAN FALSE )
+else()
   if(BLAS_FIND_REQUIRED)
     message(FATAL_ERROR "FindBLAS requires Fortran, C, or C++ to be enabled.")
   else()
@@ -136,7 +132,7 @@ if(_libraries_work)
   # Test this combination of libraries.
   set(CMAKE_REQUIRED_LIBRARIES ${_flags} ${${LIBRARIES}} ${_thread})
 #  message("DEBUG: CMAKE_REQUIRED_LIBRARIES = ${CMAKE_REQUIRED_LIBRARIES}")
-  if (CMAKE_Fortran_COMPILER_LOADED)
+  if (_CHECK_FORTRAN)
     check_fortran_function_exists("${_name}" ${_prefix}${_combined_name}_WORKS)
   else()
     check_function_exists("${_name}_" ${_prefix}${_combined_name}_WORKS)
@@ -487,7 +483,7 @@ if (BLA_VENDOR MATCHES "Intel" OR BLA_VENDOR STREQUAL "All")
  if (NOT WIN32)
   set(LM "-lm")
  endif ()
- if (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED)
+ if (_LANGUAGES_ MATCHES C OR _LANGUAGES_ MATCHES CXX)
   if(BLAS_FIND_QUIETLY OR NOT BLAS_FIND_REQUIRED)
     find_package(Threads)
   else()
@@ -709,4 +705,4 @@ else()
 endif()
 
 cmake_pop_check_state()
-set(CMAKE_FIND_LIBRARY_SUFFIXES ${_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
+set(CMAKE_FIND_LIBRARY_SUFFIXES ${_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})

exe/FaceLandmarkImg/CMakeLists.txt

@@ -2,6 +2,7 @@
 include_directories(${TBB_ROOT_DIR}/include)
 
 #OpenBlas library
+include_directories(${BLAS_ROOT_DIR})
 include_directories(../../lib/3rdParty/OpenBLAS/include)
 
 # Local libraries

exe/FaceLandmarkImg/FaceLandmarkImg.cpp

@@ -250,6 +250,8 @@ int main(int argc, char **argv)
 		open_face_rec.SetObservationVisualization(visualizer.GetVisImage());
 		open_face_rec.WriteObservationTracked();
 
+		open_face_rec.Close();
+
 		// Grabbing the next frame in the sequence
 		rgb_image = image_reader.GetNextImage();
 

exe/FaceLandmarkVid/CMakeLists.txt

@@ -2,6 +2,10 @@
 include_directories(${TBB_ROOT_DIR}/include)
 include_directories(${BLAS_ROOT_DIR})
 
+#OpenBlas library
+include_directories(${BLAS_ROOT_DIR})
+include_directories(../../lib/3rdParty/OpenBLAS/include)
+
 add_executable(FaceLandmarkVid FaceLandmarkVid.cpp)
 
 # Local libraries

exe/FaceLandmarkVidMulti/CMakeLists.txt

@@ -1,6 +1,10 @@
 #TBB library
 include_directories(${TBB_ROOT_DIR}/include)
 
+#OpenBlas library
+include_directories(${BLAS_ROOT_DIR})
+include_directories(../../lib/3rdParty/OpenBLAS/include)
+
 # Local libraries
 include_directories(${LandmarkDetector_SOURCE_DIR}/include)
 	

exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp

@@ -187,7 +187,8 @@ int main(int argc, char **argv)
 	fps_tracker.AddFrame();
 
 	int sequence_number = 0;
-
+	
+	
 	while (true) // this is not a for loop as we might also be reading from a webcam
 	{
 		// The sequence reader chooses what to open based on command line arguments provided
@@ -221,6 +222,9 @@ int main(int argc, char **argv)
 			INFO_STREAM("WARNING: using a AU detection in multiple face mode, it might not be as accurate and is experimental");
 		}
 
+		// For reporting progress
+		double reported_completion = 0;
+
 		INFO_STREAM("Starting tracking");
 		while (!rgb_image.empty())
 		{
@@ -397,6 +401,17 @@ int main(int argc, char **argv)
 				return 0;
 			}
 
+			// Reporting progress
+			if (sequence_reader.GetProgress() >= reported_completion / 10.0)
+			{
+				cout << reported_completion * 10 << "% ";
+				if (reported_completion == 10)
+				{
+					cout << endl;
+				}
+				reported_completion = reported_completion + 1;
+			}
+
 			// Update the frame count
 			frame_count++;
 
@@ -414,7 +429,11 @@ int main(int argc, char **argv)
 			active_models[model] = false;
 		}
 
+		INFO_STREAM("Closing output recorder");
+		open_face_rec.Close();
+		INFO_STREAM("Closing input reader");
 		sequence_reader.Close();
+		INFO_STREAM("Closed successfully");
 
 		sequence_number++;
 

exe/FeatureExtraction/CMakeLists.txt

@@ -1,6 +1,9 @@
 #TBB library
 include_directories(${TBB_ROOT_DIR}/include)
+
+#OpenBlas library
 include_directories(${BLAS_ROOT_DIR})
+include_directories(../../lib/3rdParty/OpenBLAS/include)
 
 add_executable(FeatureExtraction FeatureExtraction.cpp)
 

exe/FeatureExtraction/FeatureExtraction.cpp

@@ -253,8 +253,11 @@ int main(int argc, char **argv)
 
 		}
 
+		INFO_STREAM("Closing output recorder");
 		open_face_rec.Close();
+		INFO_STREAM("Closing input reader");
 		sequence_reader.Close();
+		INFO_STREAM("Closed successfully");
 
 		if (recording_params.outputAUs())
 		{

exe/releases/package_windows_executables.m

@@ -1,5 +1,5 @@
 clear;
-version = '2.0.1';
+version = '2.0.2';
 
 out_x86 = sprintf('OpenFace_%s_win_x86', version);
 out_x64 = sprintf('OpenFace_%s_win_x64', version);

gui/OpenFaceOffline/MainWindow.xaml.cs

@@ -235,7 +235,7 @@ namespace OpenFaceOffline
             // Setup recording
             RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(true, reader.IsWebcam(),
                 Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs,
-                RecordGaze, RecordHOG, RecordTracked, RecordAligned,
+                RecordGaze, RecordHOG, RecordTracked, RecordAligned, false,
                 reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetFPS());
 
             RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root);
@@ -357,7 +357,7 @@ namespace OpenFaceOffline
                 // Setup recording
                 RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(false, false,
                     Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs,
-                    RecordGaze, RecordHOG, RecordTracked, RecordAligned,
+                    RecordGaze, RecordHOG, RecordTracked, RecordAligned, true,
                     reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0);
 
                 RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root);

lib/local/CppInerop/RecorderInterop.h

@@ -58,12 +58,12 @@ namespace UtilitiesOF {
 	public:
 		RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks,
 			bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked,
-			bool output_aligned_faces, float fx, float fy, float cx, float cy, double fps_vid_out)
+			bool output_aligned_faces, bool record_bad_aligned, float fx, float fy, float cx, float cy, double fps_vid_out)
 		{
 
 			m_params = new Utilities::RecorderOpenFaceParameters(sequence, is_from_webcam, 
 				output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs,
-				output_gaze, output_hog, output_tracked, output_aligned_faces, fx, fy, cx, cy, fps_vid_out);
+				output_gaze, output_hog, output_tracked, output_aligned_faces, record_bad_aligned, fx, fy, cx, cy, fps_vid_out);
 
 		}
 

lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp

@@ -329,8 +329,9 @@ void CCNF_patch_expert::Read(ifstream &stream, std::vector<int> window_sizes, st
 	}
 
 	// In case we are using OpenBLAS, make sure it is not multi-threading as we are multi-threading outside of it
-	//goto_set_num_threads(1);
-	openblas_set_num_threads(1);
+	#ifndef __APPLE__
+		openblas_set_num_threads(1);
+	#endif
 
 	int n_sigmas = window_sizes.size();
 

lib/local/LandmarkDetector/src/CEN_patch_expert.cpp

@@ -91,9 +91,10 @@ void CEN_patch_expert::Read(ifstream &stream)
 {
 
 	// Setting up OpenBLAS
-	//goto_set_num_threads(1);
-	openblas_set_num_threads(1);
-
+	#ifndef __APPLE__
+		openblas_set_num_threads(1);
+	#endif
+	
 	// Sanity check
 	int read_type;
 

lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp

@@ -285,9 +285,11 @@ void CNN::ClearPrecomp()
 
 void CNN::Read(const string& location)
 {
-	//goto_set_num_threads(1);
-	openblas_set_num_threads(1);
-	
+
+	#ifndef __APPLE__
+		openblas_set_num_threads(1);
+	#endif
+
 	ifstream cnn_stream(location, ios::in | ios::binary);
 	if (cnn_stream.is_open())
 	{

lib/local/LandmarkDetector/src/PDM.cpp

@@ -460,6 +460,7 @@ void PDM::UpdateModelParameters(const cv::Mat_<float>& delta_p, cv::Mat_<float>&
 
 	// get the original rotation matrix	
 	cv::Vec3f eulerGlobal(params_global[1], params_global[2], params_global[3]);
+	
 	cv::Matx33f R1 = Utilities::Euler2RotationMatrix(eulerGlobal);
 
 	// construct R' = [1, -wz, wy
@@ -479,8 +480,18 @@ void PDM::UpdateModelParameters(const cv::Mat_<float>& delta_p, cv::Mat_<float>&
 
 	// Extract euler angle (through axis angle first to make sure it's legal)
 	cv::Vec3f axis_angle = Utilities::RotationMatrix2AxisAngle(R3);
+
 	cv::Vec3f euler = Utilities::AxisAngle2Euler(axis_angle);
 
+	// Temporary fix to numerical instability
+	if (isnan(euler[0]) || isnan(euler[1]) || isnan(euler[2]))
+	{
+		euler[0] = 0;
+		euler[1] = 0;
+		euler[2] = 0;
+
+	}
+
 	params_global[1] = euler[0];
 	params_global[2] = euler[1];
 	params_global[3] = euler[2];

lib/local/Utilities/src/RecorderOpenFace.cpp

@@ -292,8 +292,8 @@ void RecorderOpenFace::AlignedImageWritingTask()
 	{
 		std::pair<std::string, cv::Mat> tracked_data;
 
-		try {
-			aligned_face_queue.pop(tracked_data);
+		while (aligned_face_queue.try_pop(tracked_data))
+		{
 			bool write_success = cv::imwrite(tracked_data.first, tracked_data.second);
 
 			if (!write_success)
@@ -301,10 +301,7 @@ void RecorderOpenFace::AlignedImageWritingTask()
 				WARN_STREAM("Could not output similarity aligned image image");
 			}
 		}
-		catch (tbb::user_abort e1)
-		{
-			// This means the thread finished successfully
-		}
+		std::this_thread::sleep_for(std::chrono::milliseconds(100));
 	}
 
 }
@@ -317,10 +314,8 @@ void RecorderOpenFace::VideoWritingTask()
 
 		std::pair<std::string, cv::Mat> tracked_data;
 
-		try {
-
-			vis_to_out_queue.pop(tracked_data);
-
+		while (vis_to_out_queue.try_pop(tracked_data))
+		{
 			if (params.isSequence())
 			{
 				if (video_writer.isOpened())
@@ -337,10 +332,7 @@ void RecorderOpenFace::VideoWritingTask()
 				}
 			}
 		}
-		catch (tbb::user_abort e1)
-		{
-			// This means the thread finished successfully
-		}
+		std::this_thread::sleep_for(std::chrono::milliseconds(100));
 
 	}
 
@@ -527,22 +519,8 @@ RecorderOpenFace::~RecorderOpenFace()
 void RecorderOpenFace::Close()
 {
 	recording = false;
-
+	
 	// Make sure the recording threads complete
-	while (!vis_to_out_queue.empty())
-	{
-		std::this_thread::sleep_for(std::chrono::milliseconds(100));
-	}
-	while (!aligned_face_queue.empty())
-	{
-		std::this_thread::sleep_for(std::chrono::milliseconds(100));
-	}
-
-	// Free the waiting queues
-	vis_to_out_queue.abort();
-	aligned_face_queue.abort();
-
-	// Wait for the writing threads to finish
 	writing_threads.wait();	
 
 	hog_recorder.Close();

lib/local/Utilities/src/SequenceCapture.cpp

@@ -359,6 +359,13 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f
 	image_files.clear();
 
 	boost::filesystem::path image_directory(directory);
+
+	if (!boost::filesystem::exists(image_directory))
+	{
+		std::cout << "Provided directory does not exist: " << directory << std::endl;
+		return false;
+	}
+
 	std::vector<boost::filesystem::path> file_in_directory;
 	copy(boost::filesystem::directory_iterator(image_directory), boost::filesystem::directory_iterator(), back_inserter(file_in_directory));