diff --git a/lib/local/Utilities/include/RecorderOpenFace.h b/lib/local/Utilities/include/RecorderOpenFace.h index 69ccb386..8daa558e 100644 --- a/lib/local/Utilities/include/RecorderOpenFace.h +++ b/lib/local/Utilities/include/RecorderOpenFace.h @@ -38,6 +38,10 @@ #include "RecorderHOG.h" #include "RecorderOpenFaceParameters.h" +// For speeding up writing +#include "tbb/concurrent_queue.h" +#include "tbb/task_group.h" + // System includes #include @@ -105,6 +109,16 @@ namespace Utilities private: + // Used to keep track if the recording is still going (for the writing threads) + bool recording; + + // For keeping track of tasks + tbb::task_group writing_threads; + + // A thread that will write video output, so that the rest of the application does not block on it + void VideoWritingTask(); + void AlignedImageWritingTask(); + // Blocking copy, assignment and move operators, as it does not make sense to save to the same location RecorderOpenFace & operator= (const RecorderOpenFace& other); RecorderOpenFace & operator= (const RecorderOpenFace&& other); @@ -159,10 +173,16 @@ namespace Utilities // For video writing cv::VideoWriter video_writer; std::string media_filename; + + // Do not exceed 100MB in the concurrent queue + const int TRACKED_QUEUE_CAPACITY = 100; cv::Mat vis_to_out; + tbb::concurrent_bounded_queue > vis_to_out_queue; // For aligned face writing + const int ALIGNED_QUEUE_CAPACITY = 100; cv::Mat aligned_face; + tbb::concurrent_bounded_queue > aligned_face_queue; }; } diff --git a/lib/local/Utilities/src/RecorderHOG.cpp b/lib/local/Utilities/src/RecorderHOG.cpp index eb469ec2..ae6332d8 100644 --- a/lib/local/Utilities/src/RecorderHOG.cpp +++ b/lib/local/Utilities/src/RecorderHOG.cpp @@ -67,18 +67,26 @@ void RecorderHOG::Write() good_frame_float = -1; hog_file.write((char*)(&good_frame_float), 4); - - cv::MatConstIterator_ descriptor_it = hog_descriptor.begin(); - - for (int y = 0; y < num_cols; ++y) + if(hog_descriptor.isContinuous()) { - for (int x = 0; x < num_rows; ++x) - { - for (unsigned int o = 0; o < 31; ++o) - { + cv::Mat_ desc; + hog_descriptor.convertTo(desc, CV_32F); + hog_file.write((char*)desc.data, 4 * num_cols * num_rows * 31); + } + else + { + cv::MatConstIterator_ descriptor_it = hog_descriptor.begin(); - float hog_data = (float)(*descriptor_it++); - hog_file.write((char*)&hog_data, 4); + for (int y = 0; y < num_cols; ++y) + { + for (int x = 0; x < num_rows; ++x) + { + for (unsigned int o = 0; o < 31; ++o) + { + + float hog_data = (float)(*descriptor_it++); + hog_file.write((char*)&hog_data, 4); + } } } } diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 3d279009..966698b9 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -46,6 +46,10 @@ #include #include +// For threading +#include +#include + using namespace boost::filesystem; using namespace Utilities; @@ -142,6 +146,9 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) metadata_file << "Output image:" << this->media_filename << endl; this->media_filename = (path(record_root) / this->media_filename).string(); } + + // Start the video and image writing thread + writing_threads.run([&] {VideoWritingTask(); }); } // Prepare image recording @@ -151,9 +158,15 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) metadata_file << "Output aligned directory:" << this->aligned_output_directory << endl; this->aligned_output_directory = (path(record_root) / this->aligned_output_directory).string(); CreateDirectory(aligned_output_directory); + + // Start the video and image writing thread + writing_threads.run([&] {AlignedImageWritingTask(); }); } this->frame_number = 0; + + recording = true; + } @@ -214,6 +227,7 @@ RecorderOpenFace::RecorderOpenFace(const std::string in_filename, const Recorder } PrepareRecording(in_filename); + } RecorderOpenFace::RecorderOpenFace(const std::string in_filename, const RecorderOpenFaceParameters& parameters, std::string output_directory):video_writer(), params(parameters) @@ -255,6 +269,10 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) { video_writer.open(media_filename, CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), params.outputFps(), vis_track.size(), true); + // Set up the queue for video writing based on output size + int capacity = (1024 * 1024 * TRACKED_QUEUE_CAPACITY) / (vis_track.size().width * vis_track.size().height * vis_track.channels()); + vis_to_out_queue.set_capacity(capacity); + if (!video_writer.isOpened()) { WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN."); @@ -266,13 +284,61 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) } } - vis_to_out = vis_track; } } +void RecorderOpenFace::AlignedImageWritingTask() +{ + while (recording) + { + std::pair tracked_data; + + if (aligned_face_queue.try_pop(tracked_data)) + { + bool write_success = cv::imwrite(tracked_data.first, tracked_data.second); + + if (!write_success) + { + WARN_STREAM("Could not output similarity aligned image image"); + } + } + std::this_thread::sleep_for(std::chrono::microseconds(1000)); + + } +} + +void RecorderOpenFace::VideoWritingTask() +{ + while(recording) + { + std::pair tracked_data; + + if(vis_to_out_queue.try_pop(tracked_data)) + { + if (params.isSequence()) + { + if (video_writer.isOpened()) + { + video_writer.write(tracked_data.second); + } + } + else + { + bool out_success = cv::imwrite(tracked_data.first, tracked_data.second); + if (!out_success) + { + WARN_STREAM("Could not output tracked image"); + } + } + } + std::this_thread::sleep_for(std::chrono::microseconds(1000)); + + } +} + void RecorderOpenFace::WriteObservation() { @@ -339,35 +405,26 @@ void RecorderOpenFace::WriteObservation() std::string preferredSlash = slash.make_preferred().string(); string out_file = aligned_output_directory + preferredSlash + string(name); - bool write_success = cv::imwrite(out_file, aligned_face); - if (!write_success) - { - WARN_STREAM("Could not output similarity aligned image image"); - } + aligned_face_queue.push(std::pair(out_file, aligned_face)); + } if(params.outputTracked()) { + if (vis_to_out.empty()) { WARN_STREAM("Output tracked video frame is not set"); } - if(params.isSequence() ) + if (params.isSequence()) { - if(video_writer.isOpened()) - { - video_writer.write(vis_to_out); - } + vis_to_out_queue.push(std::pair("", vis_to_out)); } else { - bool out_success = cv::imwrite(media_filename, vis_to_out); - if (!out_success) - { - WARN_STREAM("Could not output tracked image"); - } + vis_to_out_queue.push(std::pair(media_filename, vis_to_out)); } // Clear the output vis_to_out = cv::Mat(); @@ -440,11 +497,15 @@ RecorderOpenFace::~RecorderOpenFace() void RecorderOpenFace::Close() { + recording = false; + hog_recorder.Close(); csv_recorder.Close(); video_writer.release(); metadata_file.close(); + // Wait for the writing threads to finish + //writing_threads.wait(); }