diff --git a/.travis.yml b/.travis.yml index 155b85a8..1bdf3d8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,8 @@ compiler: os: - linux - + - osx + before_install: # OpenCV dependencies and boost @@ -58,7 +59,6 @@ before_install: brew install tbb; brew tap homebrew/science; brew install opencv3; - brew install openblas; brew upgrade boost; fi @@ -78,7 +78,9 @@ script: - mkdir build - cd build - cmake -D CMAKE_BUILD_TYPE=RELEASE CMAKE_CXX_FLAGS="-std=c++11" -D CMAKE_EXE_LINKER_FLAGS="-std=c++11" .. - - make - - ../build/bin/FaceLandmarkImg -inroot ../samples -f sample1.jpg -out_dir data -of sample1.txt -multi_view 1 -wild -q + - make -j4 + - export OMP_NUM_THREADS=1 + - export VECLIB_MAXIMUM_THREADS=1 + - ../build/bin/FaceLandmarkImg -f ../samples/sample1.jpg -f ../samples/sample3.jpg -out_dir data -multi_view 1 -wild -q - ../build/bin/FeatureExtraction -fdir "../samples/image_sequence" -out_dir output -q - - ../build/bin/FaceLandmarkVidMulti -fdir ../samples/image_sequence -q -mloc model/main_clnf_general.txt \ No newline at end of file + - ../build/bin/FaceLandmarkVidMulti -fdir ../samples/image_sequence -q -mloc model/main_clm_general.txt \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 0df087d5..b28dd364 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ add_definitions(-DCONFIG_DIR="${CONFIG_DIR}") find_package( BLAS REQUIRED ) include_directories( ${BLAS_INCLUDE_DIRS} ) -LINK_DIRECTORIES(${BLAS_LIBRARY_DIRS}) +LINK_DIRECTORIES(${BLAS_LIBRARY_DIRS}) find_package( OpenCV 3.3 REQUIRED ) diff --git a/cmake/modules/FindBLAS.cmake b/cmake/modules/FindBLAS.cmake index 2684617f..603fb78d 100644 --- a/cmake/modules/FindBLAS.cmake +++ b/cmake/modules/FindBLAS.cmake @@ -1,6 +1,3 @@ -# Distributed under the OSI-approved BSD 3-Clause License. See accompanying -# file Copyright.txt or https://cmake.org/licensing for details. - #.rst: # FindBLAS # -------- @@ -32,32 +29,26 @@ # all the possibilities # BLA_F95 if set on tries to find the f95 interfaces for BLAS/LAPACK # -# List of vendors (BLA_VENDOR) valid in this module: +# ######### ## List of vendors (BLA_VENDOR) valid in this module # +# Goto,OpenBLAS,ATLAS PhiPACK,CXML,DXML,SunPerf,SCSL,SGIMATH,IBMESSL, +# Intel10_32 (intel mkl v10 32 bit),Intel10_64lp (intel mkl v10 64 bit, +# lp thread model, lp64 model), # Intel10_64lp_seq (intel mkl v10 64 +# bit,sequential code, lp64 model), # Intel( older versions of mkl 32 +# and 64 bit), ACML,ACML_MP,ACML_GPU,Apple, NAS, Generic C/CXX should be +# enabled to use Intel mkl + +#============================================================================= +# Copyright 2007-2009 Kitware, Inc. # -# * Goto -# * OpenBLAS -# * ATLAS PhiPACK -# * CXML -# * DXML -# * SunPerf -# * SCSL -# * SGIMATH -# * IBMESSL -# * Intel10_32 (intel mkl v10 32 bit) -# * Intel10_64lp (intel mkl v10 64 bit, lp thread model, lp64 model) -# * Intel10_64lp_seq (intel mkl v10 64 bit, sequential code, lp64 model) -# * Intel (older versions of mkl 32 and 64 bit) -# * ACML -# * ACML_MP -# * ACML_GPU -# * Apple -# * NAS -# * Generic -# -# .. note:: -# -# C/CXX should be enabled to use Intel mkl +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. # +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) include(${CMAKE_CURRENT_LIST_DIR}/CheckFunctionExists.cmake) include(${CMAKE_CURRENT_LIST_DIR}/CheckFortranFunctionExists.cmake) @@ -68,7 +59,12 @@ set(CMAKE_REQUIRED_QUIET ${BLAS_FIND_QUIETLY}) set(_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) # Check the language being used -if( NOT (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED OR CMAKE_Fortran_COMPILER_LOADED) ) +get_property( _LANGUAGES_ GLOBAL PROPERTY ENABLED_LANGUAGES ) +if( _LANGUAGES_ MATCHES Fortran ) + set( _CHECK_FORTRAN TRUE ) +elseif( (_LANGUAGES_ MATCHES C) OR (_LANGUAGES_ MATCHES CXX) ) + set( _CHECK_FORTRAN FALSE ) +else() if(BLAS_FIND_REQUIRED) message(FATAL_ERROR "FindBLAS requires Fortran, C, or C++ to be enabled.") else() @@ -136,7 +132,7 @@ if(_libraries_work) # Test this combination of libraries. set(CMAKE_REQUIRED_LIBRARIES ${_flags} ${${LIBRARIES}} ${_thread}) # message("DEBUG: CMAKE_REQUIRED_LIBRARIES = ${CMAKE_REQUIRED_LIBRARIES}") - if (CMAKE_Fortran_COMPILER_LOADED) + if (_CHECK_FORTRAN) check_fortran_function_exists("${_name}" ${_prefix}${_combined_name}_WORKS) else() check_function_exists("${_name}_" ${_prefix}${_combined_name}_WORKS) @@ -487,7 +483,7 @@ if (BLA_VENDOR MATCHES "Intel" OR BLA_VENDOR STREQUAL "All") if (NOT WIN32) set(LM "-lm") endif () - if (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED) + if (_LANGUAGES_ MATCHES C OR _LANGUAGES_ MATCHES CXX) if(BLAS_FIND_QUIETLY OR NOT BLAS_FIND_REQUIRED) find_package(Threads) else() @@ -709,4 +705,4 @@ else() endif() cmake_pop_check_state() -set(CMAKE_FIND_LIBRARY_SUFFIXES ${_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) +set(CMAKE_FIND_LIBRARY_SUFFIXES ${_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) \ No newline at end of file diff --git a/exe/FaceLandmarkImg/CMakeLists.txt b/exe/FaceLandmarkImg/CMakeLists.txt index 1855d1c5..e7b164f3 100644 --- a/exe/FaceLandmarkImg/CMakeLists.txt +++ b/exe/FaceLandmarkImg/CMakeLists.txt @@ -2,6 +2,7 @@ include_directories(${TBB_ROOT_DIR}/include) #OpenBlas library +include_directories(${BLAS_ROOT_DIR}) include_directories(../../lib/3rdParty/OpenBLAS/include) # Local libraries diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 14f5f064..e98bd1f4 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -250,6 +250,8 @@ int main(int argc, char **argv) open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); open_face_rec.WriteObservationTracked(); + open_face_rec.Close(); + // Grabbing the next frame in the sequence rgb_image = image_reader.GetNextImage(); diff --git a/exe/FaceLandmarkVid/CMakeLists.txt b/exe/FaceLandmarkVid/CMakeLists.txt index 0de71d26..ba0a4977 100644 --- a/exe/FaceLandmarkVid/CMakeLists.txt +++ b/exe/FaceLandmarkVid/CMakeLists.txt @@ -2,6 +2,10 @@ include_directories(${TBB_ROOT_DIR}/include) include_directories(${BLAS_ROOT_DIR}) +#OpenBlas library +include_directories(${BLAS_ROOT_DIR}) +include_directories(../../lib/3rdParty/OpenBLAS/include) + add_executable(FaceLandmarkVid FaceLandmarkVid.cpp) # Local libraries diff --git a/exe/FaceLandmarkVidMulti/CMakeLists.txt b/exe/FaceLandmarkVidMulti/CMakeLists.txt index d57379c3..b0637103 100644 --- a/exe/FaceLandmarkVidMulti/CMakeLists.txt +++ b/exe/FaceLandmarkVidMulti/CMakeLists.txt @@ -1,6 +1,10 @@ #TBB library include_directories(${TBB_ROOT_DIR}/include) +#OpenBlas library +include_directories(${BLAS_ROOT_DIR}) +include_directories(../../lib/3rdParty/OpenBLAS/include) + # Local libraries include_directories(${LandmarkDetector_SOURCE_DIR}/include) diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index 2ab0b2ac..2da12471 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -187,7 +187,8 @@ int main(int argc, char **argv) fps_tracker.AddFrame(); int sequence_number = 0; - + + while (true) // this is not a for loop as we might also be reading from a webcam { // The sequence reader chooses what to open based on command line arguments provided @@ -221,6 +222,9 @@ int main(int argc, char **argv) INFO_STREAM("WARNING: using a AU detection in multiple face mode, it might not be as accurate and is experimental"); } + // For reporting progress + double reported_completion = 0; + INFO_STREAM("Starting tracking"); while (!rgb_image.empty()) { @@ -397,6 +401,17 @@ int main(int argc, char **argv) return 0; } + // Reporting progress + if (sequence_reader.GetProgress() >= reported_completion / 10.0) + { + cout << reported_completion * 10 << "% "; + if (reported_completion == 10) + { + cout << endl; + } + reported_completion = reported_completion + 1; + } + // Update the frame count frame_count++; @@ -414,7 +429,11 @@ int main(int argc, char **argv) active_models[model] = false; } + INFO_STREAM("Closing output recorder"); + open_face_rec.Close(); + INFO_STREAM("Closing input reader"); sequence_reader.Close(); + INFO_STREAM("Closed successfully"); sequence_number++; diff --git a/exe/FeatureExtraction/CMakeLists.txt b/exe/FeatureExtraction/CMakeLists.txt index 8ca9eb9c..e69b3443 100644 --- a/exe/FeatureExtraction/CMakeLists.txt +++ b/exe/FeatureExtraction/CMakeLists.txt @@ -1,6 +1,9 @@ #TBB library include_directories(${TBB_ROOT_DIR}/include) + +#OpenBlas library include_directories(${BLAS_ROOT_DIR}) +include_directories(../../lib/3rdParty/OpenBLAS/include) add_executable(FeatureExtraction FeatureExtraction.cpp) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 4e85ff42..e5479e04 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -253,8 +253,11 @@ int main(int argc, char **argv) } + INFO_STREAM("Closing output recorder"); open_face_rec.Close(); + INFO_STREAM("Closing input reader"); sequence_reader.Close(); + INFO_STREAM("Closed successfully"); if (recording_params.outputAUs()) { diff --git a/exe/releases/package_windows_executables.m b/exe/releases/package_windows_executables.m index 05c6232a..d19bb256 100644 --- a/exe/releases/package_windows_executables.m +++ b/exe/releases/package_windows_executables.m @@ -1,5 +1,5 @@ clear; -version = '2.0.1'; +version = '2.0.2'; out_x86 = sprintf('OpenFace_%s_win_x86', version); out_x64 = sprintf('OpenFace_%s_win_x64', version); diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index 9008df65..1531c357 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -235,7 +235,7 @@ namespace OpenFaceOffline // Setup recording RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(true, reader.IsWebcam(), Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs, - RecordGaze, RecordHOG, RecordTracked, RecordAligned, + RecordGaze, RecordHOG, RecordTracked, RecordAligned, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetFPS()); RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root); @@ -357,7 +357,7 @@ namespace OpenFaceOffline // Setup recording RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(false, false, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs, - RecordGaze, RecordHOG, RecordTracked, RecordAligned, + RecordGaze, RecordHOG, RecordTracked, RecordAligned, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0); RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root); diff --git a/lib/local/CppInerop/RecorderInterop.h b/lib/local/CppInerop/RecorderInterop.h index 8a710ae3..95e9544b 100644 --- a/lib/local/CppInerop/RecorderInterop.h +++ b/lib/local/CppInerop/RecorderInterop.h @@ -58,12 +58,12 @@ namespace UtilitiesOF { public: RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked, - bool output_aligned_faces, float fx, float fy, float cx, float cy, double fps_vid_out) + bool output_aligned_faces, bool record_bad_aligned, float fx, float fy, float cx, float cy, double fps_vid_out) { m_params = new Utilities::RecorderOpenFaceParameters(sequence, is_from_webcam, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, - output_gaze, output_hog, output_tracked, output_aligned_faces, fx, fy, cx, cy, fps_vid_out); + output_gaze, output_hog, output_tracked, output_aligned_faces, record_bad_aligned, fx, fy, cx, cy, fps_vid_out); } diff --git a/lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp b/lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp index 00972202..37ee7d91 100644 --- a/lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp +++ b/lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp @@ -329,8 +329,9 @@ void CCNF_patch_expert::Read(ifstream &stream, std::vector window_sizes, st } // In case we are using OpenBLAS, make sure it is not multi-threading as we are multi-threading outside of it - //goto_set_num_threads(1); - openblas_set_num_threads(1); + #ifndef __APPLE__ + openblas_set_num_threads(1); + #endif int n_sigmas = window_sizes.size(); diff --git a/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp b/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp index 455e8309..c4b18571 100644 --- a/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp +++ b/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp @@ -91,9 +91,10 @@ void CEN_patch_expert::Read(ifstream &stream) { // Setting up OpenBLAS - //goto_set_num_threads(1); - openblas_set_num_threads(1); - + #ifndef __APPLE__ + openblas_set_num_threads(1); + #endif + // Sanity check int read_type; diff --git a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp index 4a790e0d..46681bf4 100644 --- a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp +++ b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp @@ -285,9 +285,11 @@ void CNN::ClearPrecomp() void CNN::Read(const string& location) { - //goto_set_num_threads(1); - openblas_set_num_threads(1); - + + #ifndef __APPLE__ + openblas_set_num_threads(1); + #endif + ifstream cnn_stream(location, ios::in | ios::binary); if (cnn_stream.is_open()) { diff --git a/lib/local/LandmarkDetector/src/PDM.cpp b/lib/local/LandmarkDetector/src/PDM.cpp index 8baed710..12cc9333 100644 --- a/lib/local/LandmarkDetector/src/PDM.cpp +++ b/lib/local/LandmarkDetector/src/PDM.cpp @@ -460,6 +460,7 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& // get the original rotation matrix cv::Vec3f eulerGlobal(params_global[1], params_global[2], params_global[3]); + cv::Matx33f R1 = Utilities::Euler2RotationMatrix(eulerGlobal); // construct R' = [1, -wz, wy @@ -479,8 +480,18 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& // Extract euler angle (through axis angle first to make sure it's legal) cv::Vec3f axis_angle = Utilities::RotationMatrix2AxisAngle(R3); + cv::Vec3f euler = Utilities::AxisAngle2Euler(axis_angle); + // Temporary fix to numerical instability + if (isnan(euler[0]) || isnan(euler[1]) || isnan(euler[2])) + { + euler[0] = 0; + euler[1] = 0; + euler[2] = 0; + + } + params_global[1] = euler[0]; params_global[2] = euler[1]; params_global[3] = euler[2]; diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 4500a937..4aa9f7ca 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -292,8 +292,8 @@ void RecorderOpenFace::AlignedImageWritingTask() { std::pair tracked_data; - try { - aligned_face_queue.pop(tracked_data); + while (aligned_face_queue.try_pop(tracked_data)) + { bool write_success = cv::imwrite(tracked_data.first, tracked_data.second); if (!write_success) @@ -301,10 +301,7 @@ void RecorderOpenFace::AlignedImageWritingTask() WARN_STREAM("Could not output similarity aligned image image"); } } - catch (tbb::user_abort e1) - { - // This means the thread finished successfully - } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } @@ -317,10 +314,8 @@ void RecorderOpenFace::VideoWritingTask() std::pair tracked_data; - try { - - vis_to_out_queue.pop(tracked_data); - + while (vis_to_out_queue.try_pop(tracked_data)) + { if (params.isSequence()) { if (video_writer.isOpened()) @@ -337,10 +332,7 @@ void RecorderOpenFace::VideoWritingTask() } } } - catch (tbb::user_abort e1) - { - // This means the thread finished successfully - } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } @@ -527,22 +519,8 @@ RecorderOpenFace::~RecorderOpenFace() void RecorderOpenFace::Close() { recording = false; - + // Make sure the recording threads complete - while (!vis_to_out_queue.empty()) - { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - while (!aligned_face_queue.empty()) - { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - // Free the waiting queues - vis_to_out_queue.abort(); - aligned_face_queue.abort(); - - // Wait for the writing threads to finish writing_threads.wait(); hog_recorder.Close(); diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index 3d8da139..20839bb8 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -359,6 +359,13 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f image_files.clear(); boost::filesystem::path image_directory(directory); + + if (!boost::filesystem::exists(image_directory)) + { + std::cout << "Provided directory does not exist: " << directory << std::endl; + return false; + } + std::vector file_in_directory; copy(boost::filesystem::directory_iterator(image_directory), boost::filesystem::directory_iterator(), back_inserter(file_in_directory));