From 562b97db70781c06f489244e34d2efee915beb7d Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sat, 28 Apr 2018 22:22:53 +0100 Subject: [PATCH 1/5] Starting multi-threading in sequence captures. --- lib/local/Utilities/include/SequenceCapture.h | 21 ++++ lib/local/Utilities/src/RecorderOpenFace.cpp | 22 +++- lib/local/Utilities/src/SequenceCapture.cpp | 113 ++++++++++++------ 3 files changed, 113 insertions(+), 43 deletions(-) diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h index daafbacf..143a3566 100644 --- a/lib/local/Utilities/include/SequenceCapture.h +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -39,6 +39,10 @@ #include #include +// For speeding up capture +#include "tbb/concurrent_queue.h" +#include "tbb/task_group.h" + // OpenCV includes #include #include @@ -109,6 +113,17 @@ namespace Utilities private: + // For faster input, multi-thread the capture so it is not waiting for processing to be done + + // Used to keep track if the recording is still going (for the writing threads) + bool capturing; + + // For keeping track of tasks + tbb::task_group capture_threads; + + // A thread that will write video output, so that the rest of the application does not block on it + void CaptureThread(); + // Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source, and this would cause issues, especially with webcams SequenceCapture & operator= (const SequenceCapture& other); SequenceCapture & operator= (const SequenceCapture&& other); @@ -122,6 +137,12 @@ namespace Utilities cv::Mat latest_frame; cv::Mat_ latest_gray_frame; + + // Storing the captured data queue + const int CAPTURE_CAPACITY = 200; // 200 MB + // Storing capture timestamp, RGB image, gray image + tbb::concurrent_bounded_queue > > capture_queue; + // Keeping track of frame number and the files in the image sequence size_t frame_num; std::vector image_files; diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 0c6730a0..439450ed 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -129,7 +129,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) hog_filename = (path(record_root) / hog_filename).string(); hog_recorder.Open(hog_filename); } - + // saving the videos if (params.outputTracked()) { @@ -158,7 +158,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) metadata_file << "Output aligned directory:" << this->aligned_output_directory << endl; this->aligned_output_directory = (path(record_root) / this->aligned_output_directory).string(); CreateDirectory(aligned_output_directory); - + // Start the video and image writing thread writing_threads.run([&] {AlignedImageWritingTask(); }); } @@ -292,6 +292,7 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) void RecorderOpenFace::AlignedImageWritingTask() { + while (recording) { std::pair tracked_data; @@ -305,9 +306,11 @@ void RecorderOpenFace::AlignedImageWritingTask() WARN_STREAM("Could not output similarity aligned image image"); } } - std::this_thread::sleep_for(std::chrono::microseconds(1000)); + std::this_thread::sleep_for(std::chrono::microseconds(10000)); + } + } void RecorderOpenFace::VideoWritingTask() @@ -334,9 +337,11 @@ void RecorderOpenFace::VideoWritingTask() } } } - std::this_thread::sleep_for(std::chrono::microseconds(1000)); + //cout << "Vis size:" << vis_to_out_queue.size() << endl; + std::this_thread::sleep_for(std::chrono::microseconds(10000)); } + } void RecorderOpenFace::WriteObservation() @@ -391,6 +396,12 @@ void RecorderOpenFace::WriteObservation() // Write aligned faces if (params.outputAlignedFaces()) { + if (frame_number == 1) + { + int capacity = (1024 * 1024 * ALIGNED_QUEUE_CAPACITY) / (aligned_face.size().width *aligned_face.size().height * aligned_face.channels()); + aligned_face_queue.set_capacity(capacity); + } + char name[100]; // Filename is based on frame number @@ -407,6 +418,9 @@ void RecorderOpenFace::WriteObservation() string out_file = aligned_output_directory + preferredSlash + string(name); aligned_face_queue.push(std::pair(out_file, aligned_face)); + + // Clear the image + aligned_face = cv::Mat(); } diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index a0a7b1e6..d0181def 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -268,6 +268,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, this->name = "webcam_" + time; start_time = cv::getTickCount(); + capturing = true; + capture_threads.run([&] {CaptureThread(); }); return true; @@ -275,8 +277,14 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, void SequenceCapture::Close() { + // Close the capturing threads + capturing = false; + capture_threads.wait(); + + // Release the capture objects if (capture.isOpened()) capture.release(); + } // Destructor that releases the capture @@ -325,6 +333,8 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, SetCameraIntrinsics(fx, fy, cx, cy); this->name = video_file; + capturing = true; + capture_threads.run([&] {CaptureThread(); }); return true; @@ -381,6 +391,8 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f is_webcam = false; is_image_seq = true; vid_length = image_files.size(); + capturing = true; + capture_threads.run([&] {CaptureThread(); }); return true; @@ -415,47 +427,70 @@ void SequenceCapture::SetCameraIntrinsics(float fx, float fy, float cx, float cy } } +void SequenceCapture::CaptureThread() +{ + int capacity = (CAPTURE_CAPACITY * 1024 * 1024) / (4 * frame_width * frame_height); + capture_queue.set_capacity(capacity); + int frame_num_int = 0; + + while(capturing) + { + double timestamp_curr = 0; + cv::Mat tmp_frame; + cv::Mat_ tmp_gray_frame; + + if (is_webcam || !is_image_seq) + { + bool success = capture.read(tmp_frame); + + if (!success) + { + // Indicate lack of success by returning an empty image + tmp_frame = cv::Mat(); + capturing = false; + } + + // Recording the timestamp + if (!is_webcam) + { + timestamp_curr = frame_num_int * (1.0 / fps); + } + else + { + timestamp_curr = (cv::getTickCount() - start_time) / cv::getTickFrequency(); + } + + } + else if (is_image_seq) + { + if (image_files.empty() || frame_num_int >= image_files.size()) + { + // Indicate lack of success by returning an empty image + tmp_frame = cv::Mat(); + capturing = false; + } + else + { + tmp_frame = cv::imread(image_files[frame_num_int], CV_LOAD_IMAGE_COLOR); + } + timestamp_curr = 0; + } + + frame_num_int++; + // Set the grayscale frame + ConvertToGrayscale_8bit(tmp_frame, tmp_gray_frame); + + capture_queue.push(std::make_tuple(timestamp_curr, tmp_frame, tmp_gray_frame)); + } +} + cv::Mat SequenceCapture::GetNextFrame() { - - if (is_webcam || !is_image_seq) - { - - bool success = capture.read(latest_frame); - - if (!success) - { - // Indicate lack of success by returning an empty image - latest_frame = cv::Mat(); - } - - // Recording the timestamp - if (!is_webcam) - { - time_stamp = frame_num * (1.0 / fps); - } - else - { - time_stamp = (cv::getTickCount() - start_time) / cv::getTickFrequency(); - } - - } - else if (is_image_seq) - { - if (image_files.empty() || frame_num >= image_files.size()) - { - // Indicate lack of success by returning an empty image - latest_frame = cv::Mat(); - } - else - { - latest_frame = cv::imread(image_files[frame_num], CV_LOAD_IMAGE_COLOR); - } - time_stamp = 0; - } - - // Set the grayscale frame - ConvertToGrayscale_8bit(latest_frame, latest_gray_frame); + std::tuple > data; + capture_queue.pop(data); + time_stamp = std::get<0>(data); + latest_frame = std::get<1>(data); + latest_gray_frame = std::get<2>(data); frame_num++; From a4e20c0454b24912f2209eaf665b0b303259c6c1 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sun, 29 Apr 2018 13:19:33 +0100 Subject: [PATCH 2/5] Making sure sequence reading is finished before new sequence is started, making sure the right number of threads is set for OpenBLAS --- exe/FaceLandmarkVid/FaceLandmarkVid.cpp | 1 + exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp | 2 ++ exe/FeatureExtraction/FeatureExtraction.cpp | 1 + .../LandmarkDetector/src/CCNF_patch_expert.cpp | 1 + lib/local/LandmarkDetector/src/CEN_patch_expert.cpp | 3 ++- .../LandmarkDetector/src/FaceDetectorMTCNN.cpp | 1 + lib/local/Utilities/src/RecorderOpenFace.cpp | 13 ++++++++----- lib/local/Utilities/src/SequenceCapture.cpp | 7 ++++++- 8 files changed, 22 insertions(+), 7 deletions(-) diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index 0ff38aea..76e0fdf1 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -186,6 +186,7 @@ int main(int argc, char **argv) // Reset the model, for the next video face_model.Reset(); + sequence_reader.Close(); sequence_number++; diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index 7fd62185..b49e6531 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -407,6 +407,8 @@ int main(int argc, char **argv) active_models[model] = false; } + sequence_reader.Close(); + sequence_number++; } diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 6e9e78b1..74a08624 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -262,6 +262,7 @@ int main(int argc, char **argv) } open_face_rec.Close(); + sequence_reader.Close(); if (recording_params.outputAUs()) { diff --git a/lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp b/lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp index 520a5ea7..baabd730 100644 --- a/lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp +++ b/lib/local/LandmarkDetector/src/CCNF_patch_expert.cpp @@ -333,6 +333,7 @@ void CCNF_patch_expert::Read(ifstream &stream, std::vector window_sizes, st } // In case we are using OpenBLAS, make sure it is not multi-threading as we are multi-threading outside of it + goto_set_num_threads(1); openblas_set_num_threads(1); int n_sigmas = window_sizes.size(); diff --git a/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp b/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp index b2b8accd..6da7c923 100644 --- a/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp +++ b/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp @@ -95,8 +95,9 @@ void CEN_patch_expert::Read(ifstream &stream) { // Setting up OpenBLAS + goto_set_num_threads(1); openblas_set_num_threads(1); - + // Sanity check int read_type; diff --git a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp index 8b85c88e..65b14f1d 100644 --- a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp +++ b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp @@ -289,6 +289,7 @@ void CNN::ClearPrecomp() void CNN::Read(const string& location) { + goto_set_num_threads(1); openblas_set_num_threads(1); ifstream cnn_stream(location, ios::in | ios::binary); diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 439450ed..83758bd3 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -306,9 +306,10 @@ void RecorderOpenFace::AlignedImageWritingTask() WARN_STREAM("Could not output similarity aligned image image"); } } - - std::this_thread::sleep_for(std::chrono::microseconds(10000)); - + else + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } } } @@ -337,8 +338,10 @@ void RecorderOpenFace::VideoWritingTask() } } } - //cout << "Vis size:" << vis_to_out_queue.size() << endl; - std::this_thread::sleep_for(std::chrono::microseconds(10000)); + else + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } } diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index d0181def..eee1ed5f 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -262,7 +262,7 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, INFO_STREAM("FPS of the webcam cannot be determined, assuming 30"); fps = 30; } - + SetCameraIntrinsics(fx, fy, cx, cy); std::string time = currentDateTime(); this->name = "webcam_" + time; @@ -279,6 +279,11 @@ void SequenceCapture::Close() { // Close the capturing threads capturing = false; + + // In case the queue is full and the thread is blocking, try popping to release it + std::tuple > data; + capture_queue.try_pop(data); + capture_threads.wait(); // Release the capture objects From 9766a93e9ac8dda036f3b18c71c3db801e5f6fff Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sun, 29 Apr 2018 18:48:04 +0100 Subject: [PATCH 3/5] Bug fixes with multi-threaded sequence capture and recording. Fix with multi-hypothesis visualization in matlab. --- lib/local/Utilities/src/RecorderOpenFace.cpp | 62 ++++++++++++------- lib/local/Utilities/src/SequenceCapture.cpp | 5 +- .../fitting/Fitting_from_bb_multi_hyp.m | 2 +- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 83758bd3..aac146e3 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -82,6 +82,8 @@ void CreateDirectory(std::string output_path) void RecorderOpenFace::PrepareRecording(const std::string& in_filename) { + recording = true; + // Construct the directories required for the output CreateDirectory(record_root); @@ -129,7 +131,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) hog_filename = (path(record_root) / hog_filename).string(); hog_recorder.Open(hog_filename); } - + // saving the videos if (params.outputTracked()) { @@ -146,7 +148,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) metadata_file << "Output image:" << this->media_filename << endl; this->media_filename = (path(record_root) / this->media_filename).string(); } - + // Start the video and image writing thread writing_threads.run([&] {VideoWritingTask(); }); } @@ -164,10 +166,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) } this->frame_number = 0; - - recording = true; - - + } RecorderOpenFace::RecorderOpenFace(const std::string in_filename, const RecorderOpenFaceParameters& parameters, std::vector& arguments):video_writer(), params(parameters) @@ -269,10 +268,6 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) { video_writer.open(media_filename, CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), params.outputFps(), vis_track.size(), true); - // Set up the queue for video writing based on output size - int capacity = (1024 * 1024 * TRACKED_QUEUE_CAPACITY) / (vis_track.size().width * vis_track.size().height * vis_track.channels()); - vis_to_out_queue.set_capacity(capacity); - if (!video_writer.isOpened()) { WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN."); @@ -293,12 +288,12 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) void RecorderOpenFace::AlignedImageWritingTask() { - while (recording) + while (recording || !aligned_face_queue.empty()) { std::pair tracked_data; - if (aligned_face_queue.try_pop(tracked_data)) - { + try { + aligned_face_queue.pop(tracked_data); bool write_success = cv::imwrite(tracked_data.first, tracked_data.second); if (!write_success) @@ -306,9 +301,9 @@ void RecorderOpenFace::AlignedImageWritingTask() WARN_STREAM("Could not output similarity aligned image image"); } } - else + catch (tbb::user_abort e1) { - std::this_thread::sleep_for(std::chrono::milliseconds(50)); + // This means the thread finished successfully } } @@ -316,12 +311,16 @@ void RecorderOpenFace::AlignedImageWritingTask() void RecorderOpenFace::VideoWritingTask() { - while(recording) + + while(recording || !vis_to_out_queue.empty()) { + std::pair tracked_data; - if(vis_to_out_queue.try_pop(tracked_data)) - { + try { + + vis_to_out_queue.pop(tracked_data); + if (params.isSequence()) { if (video_writer.isOpened()) @@ -338,9 +337,9 @@ void RecorderOpenFace::VideoWritingTask() } } } - else + catch (tbb::user_abort e1) { - std::this_thread::sleep_for(std::chrono::milliseconds(50)); + // This means the thread finished successfully } } @@ -430,6 +429,13 @@ void RecorderOpenFace::WriteObservation() if(params.outputTracked()) { + if (frame_number == 1) + { + // Set up the queue for video writing based on output size + int capacity = (1024 * 1024 * TRACKED_QUEUE_CAPACITY) / (vis_to_out.size().width * vis_to_out.size().height * vis_to_out.channels()); + vis_to_out_queue.set_capacity(capacity); + } + if (vis_to_out.empty()) { WARN_STREAM("Output tracked video frame is not set"); @@ -516,8 +522,22 @@ void RecorderOpenFace::Close() { recording = false; + // Make sure the recording threads complete + while (!vis_to_out_queue.empty()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + while (!aligned_face_queue.empty()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + // Free the waiting queues + vis_to_out_queue.abort(); + aligned_face_queue.abort(); + // Wait for the writing threads to finish - writing_threads.wait(); + writing_threads.wait(); hog_recorder.Close(); csv_recorder.Close(); diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index eee1ed5f..3b606c3e 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -280,11 +280,14 @@ void SequenceCapture::Close() // Close the capturing threads capturing = false; - // In case the queue is full and the thread is blocking, try popping to release it + // In case the queue is full and the thread is blocking, free one element so it can finish std::tuple > data; capture_queue.try_pop(data); capture_threads.wait(); + + // Empty the capture queue + capture_queue.clear(); // Release the capture objects if (capture.isOpened()) diff --git a/matlab_version/fitting/Fitting_from_bb_multi_hyp.m b/matlab_version/fitting/Fitting_from_bb_multi_hyp.m index 8e018773..98de20b2 100644 --- a/matlab_version/fitting/Fitting_from_bb_multi_hyp.m +++ b/matlab_version/fitting/Fitting_from_bb_multi_hyp.m @@ -89,7 +89,7 @@ function [ shape2D, global_params, local_params, final_lhood, landmark_lhoods, v local_params = locals{v_ind}; shape2D = shapes(:,:,v_ind); - view_used = views_used(v); + view_used = views_used(v_ind); end From 1387710f39fdb86c8c9d5471593011ee076bbe37 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sun, 29 Apr 2018 18:49:01 +0100 Subject: [PATCH 4/5] Bug fixes with multi-threaded sequence capture and recording. Fix with multi-hypothesis visualization in matlab. --- .../menpo_challenge_helpers/conversion.mat | Bin 0 -> 18822 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 matlab_version/experiments_menpo/menpo_challenge_helpers/conversion.mat diff --git a/matlab_version/experiments_menpo/menpo_challenge_helpers/conversion.mat b/matlab_version/experiments_menpo/menpo_challenge_helpers/conversion.mat new file mode 100644 index 0000000000000000000000000000000000000000..6f293347eaf5598d99013aafb2d3bfdeeebe3053 GIT binary patch literal 18822 zcma&MRZyHw7cQC*Ah=s_4eoA3U~t#q?(S~E9fG^Ny9a{1yE_c-Fu))?|F>)3o;qFC zRqsu&Ue>+de%2!=tST!kLdwa+LMkV$%4lY3W6D6PXk+MNX76OnN2&-^my+k=U?2rL znHsv7nvmMt@sWxgZ*WDyJlJ)b_=5x_?OPZku_|74w~`Esaw5)PF{yI~ZxQqXI6(xN*?{+`FB%1pN6 zOpwDY{heAIuIEO@9MckWGdG?TBXVygM_2AsJW*AQEkj|$iKHyWk*sRHso(B@p7p*d z=-mp+;?GSt&$&u>&b@_nnM_0tX#wXr{B)mdSqEcfdPVVc!_8eNO;-#r^?=MD8wmsxE9x z2-?UhW{keFO-}h(s#LSiA2*a)Q}6axvN}oEFY+U_^VhRnpTx``lNEvECI@63ClGRk zz4CMp)TCZglUv^14h3krfF+lN<{qAc-q^(OF*{zytv1ojDTz?7SK!6$H~86C!e-Xk zCdl4xP_mY&1{QD!!UFaw!=zp%76>;p)nNk#y`2ejyp)r^#WL!6 z+jYFKj#MUIc<>yt`PH-Y?sz5Obris{-Xh!xuOBXylyYN{o~y>ibd%>pcsCz4ge2Y& ziVoIGhRxmB{c_!^q#ym$ci@Tql-3E3Zg1ibMov?ZC3dYtmh-n&r+19LkqxAvk$_7R z34_Bi%XvFm6_Bv|w&J-JDJ|D*#^}v_?EggLQ)`++MUVbIt;NkA=Adk_;$%2GL+k#bGww(^zpdzQWUD$9HSxS!Dc+lF;3su?a8#Ry36G-=EH6+kqX zYFlh$J@3#~=ZKJdmKKgsN-Po4jFu&Vw&#Yr7qr-Attwxh8bQD_%u$*eps;zySn5}6 z_Uh%`G2nY*)Y<=J<(?Oy8w5S!+;l&oWpaE%&87ROjhptfMCkYF{jW*annu$sLTyA! z{AmGt!?@JZLQX`s<9BgQB8{c%BfcER?1xiB-r@%rNbG5Ad!S7g_tg*xlFH(rON>vVw0nDiWTSmyf@8fD zUVVde=F^ZX3{EtzL}fnt-F;Z zXz~E+PR}8B{6LFnGL%%w!bMjB{=1nol)IYhuTxiy_DPV-cA7HTo|z(#ZudMicwO#Md69ooUIyyGx7c^ge?FrPZyQ3iM^b4 zFztCo`(!fT(1`){Y3#XZ-z6CHeO?D!gU4teZZp`U1i64nCx{IU%86+oueOp2RZvKt z*w9v|ri;eR!~4p~nR6`)DtUG)*kT*MMSV1K(i9Gp9(PlF%iNVaVcP!6##|Co6?)%k z^%AKHHg%vEFgvE-38V=cpm`c7eJWe2LVv^>+H0PvrhW*5CcPmQ)oaH+4ElIY|93W8 z-ELauH0xXI=S`CLf+_Akn9HczSJsQ4d&)QytE0|gp5DTJ>@pOH@q!_|a*z%*PFXmz z!QW3Bh2l=vEt}~Xg%I>XjgPwK58j4}+x8mvuNM09jO>?rAF}+97CeFzr^X546cL+Jnx0#~wJGKZ^ce?d5Lf9yefA`)riQpUE ze09gpMq=m9`WwVj=#uFC{RR^fSC*04=@47+Tc;v_ZGk84TwS56={Hqhr!V>h-$57^ zuy8+E>~Qo)Wr7)C4bzBzxJ0{A1M1#yh}Ed4K!!DLzRZ3w0vYJJ72NGF`qR@(@*P)A zZKF!18hU7ps^*=}Mm$3c1duX~VPR2h!&kBDoVkEIu83QyiTSF5oRY&r`yIPl7YP2o zZg$Pb^dMRnm9bIjOV1khrn*Pl<&TtvI?V-h)GQJp9^)`L&!w#^>50-wJD19N&cN{- z3K0VmyWs70;}vaC7LotJ?;AVRG840o02^4hl*`-n&~s@(ND@Ip&?+!{8-*vFp_ z^1Q&~F;n(~bblRV%|1$85JziUvwo5a3BG(3){wYHLTTIbt>t(8Az z@*+mRumu{m>}Y|((vCg!JiZ)^xqqkle$RtYfgkLMbHP&#Oo_og?8gl)lFy3Mb1DZ< zjEq}gV9%)Y6&6iim6ruw@OLVj#xSC!;0;X66d zmrt$2cL2d4$&JX> z(m`Fu!n>X?OkGQcVlY4pMVu5JuD4!N0?d5FmR*bh(bd0bHJ(fW(VWVoV%fVc9@oS2 zT$9=xb@m~^KfFi{(;97^DuSeJpk1$9PU^lFtTSsm-d{ zJlyu#iStdl*uoQoNSj@jU%Q{d9P{~7^!7{SE}WBpIHa-Q)GrE~@AvnWCbK}bg}p%q zFu_VYvM$$)oTg-$9 z9;+1WL+M(NQQu^jg2vZ^>2vyRiN)iedEvJw|J)eoupF+sjnEL$+vkr!bFTH7H!~3y zZF866X-UKtx(Jj#CEZFL7#+|1TV~n8pS$K>o}XykFFJQFPF*pS`nRZC=}WMz|JW|c z+>Mo{fa@PkE~R}d1T&wx;%8=)#ONY!BRjj{_}pEc=>M*ij~61g3S&sovrLVb{jIm6 za)*}~^z71i{T{A%gg-fpG?W^dETe-2$( z&HelAd6vnYz$joG^;sSS*~lhBk~k?2q2q%nP4o0KtYx6`Bg5^R$rS*F6!!Z2V*aJK zHX`h5$}o;m(?U`5MiJEOm;V*B19eZ11#7&Tb3T7_?)I z#UhA6Wk-fY8`F}GgnH2y9Nn%IE*V5QGTP!jJP}vK);18>HA5(4zS_HHN^F{B+j+`4 z2Mn3?Jn{s+n$6^W;Cg%0H1)-GY}dvlmMZPRaXCLv#clT27d0@MDPiDZgxMd*qmB+m z0pWvO1!o;iB`^=aRKio2@xJ=RMTSFfQjLA-uw&r}E&aYHB)}{>yk6!}F`bLU zXa6^;4I7qR*C8&d-&ZlGbTg2H>Lu}VR1>KS(=x4kaFCD@7Fg(Sngz&hI zO>?JA$*C5-tIh1-L@!s^C5toS*iBvp z1}b=p>Fi^cslO?=@DJq_QSX#8IbZojE|QZ;79QI-I=r93D6W*3TiVA^Dxk>qsXU22 z_V==0)SOb_Lv227f9z2~zeXSFUHoxdinxzKUD^cv?YeWYkk(`#oH}U0Jw&d3+5GWd zqHyC#Dnihi+wIh%wyrGd(>HNibF(L@Bz8(97%S0Xn9u?0`!2y4rP~JIT%=pmyGk{T ziWi-2K+vj8R}?KD00K1$J6-d~P#gCMRaQy&qceGU)~TUYd$CDWk&Qf>5mxkTd&XH3 zD#)J9IS?day*ez5%Pef?EiL)hJpstnwg>8Rogde=p8fE(eZap_@2sC;9QGb)`2oj9 z8a;lDN=vlE7Ih1MVMse^)s}3>#WYhak$g^nb?1%$PlHO^`>IiOx6g}tEAxd1ukqcB z;%1p%G#2gXLrok7U&4SsBJBd|ECH_aYKj=Mp*D*hpSQW&E-y<7Eu$_G4J*ud{n|+G z+IZM=V#c*agvLxA1Kx62v~sEWWcB5JM%30sCgE>qr&SWt*qE`I+)6x=9%)ax+mkhvJo=`G{J$4t_AeVY-nogVl}3dg+6m@VURs@-7?=`zSE zR!fi1f)|VoQE*P0+`Y*9bvl%wa^g?wQbQ{H8aQTC6i6Eqq7qV*24mDonJEP~DYUD& zGn|w&Bl?9OU0}NSJIIIniVS5=T2dDj&XQS@)9P22z9+2-n>DGlwb9FzZyA9^g}hk6 zR+QQL@t~)kj~z=Z`5Ah>{@wM@Zl;n#86%b!wa=C>k5rN5CxCWw8M1yyl)>!@;od*v zj?M#eBP`cA;~FY!S1*s>)(&3XuD0a3%9^%Df23?XZ5I%7K8`GQ%>U~PvH z%}x~!3_~X>`iVi}B|2Ni+xw<~!NKfCEFao-JV`PIlLTJhcxTj89gZ{K7VkLZ5);$XuXqFL7ISLT0#C9$WJ z*%Y86j+<-pF~$TVw{0kQAM09_z;*s`#@1-!={Q}0DJYIeOIiz7nMwC^yEzI{cmO5Ne;T!)ow&ugzj6eYyC14P}_;wp|mU_t=!O zuuljB<{2lL)w2ky7`RKFtC)-P$TJfqe{*d%h=1qs9Q=8?bt{yPcRkP9x8aiKnajdI zdAcI+=AYYi|C+{+Dr|=|d|O>!6V<3QR>TG!p7Lzais0d>V-l%5MEvvFUKwHIK1wsG z*RI>~fbgHsx$6?;+H&4>%fe^bLt&BZeKY)Sb0?8<5!d56x4;uf;|v%xNRx85o;~}? zKrM%JdgQ*DG?zQGvGyUWvIiMMu9$b71m#IE^p8vMGV{ zf{2gU%!Jiex{-|g{Kvyl*20=0ESJord|Ar`GpvtSLCBTgZLzBkb|KsOtWd?)SZ6)< zV@DQ;zqFo`ykGF(su(&q{xE%l4jR{R*Jq#Iq9%3FRQ&8Bt1rkl)P&^;8@zk&RbcGS ze-q%XMXN?VVLNE@a2vWI{Zu~h5c9oN|J*sznXqG0=Z2-q2E{%tK9r_++2EF$J7Ada zBi!>33Snao%gA=zqp{gzU@$R6s{b5^L5uxB$<@DwCHGXe@zKQZ%WYR_b-VI7I%MOZ z_#nkX{-!s}DqHmXK+y0~QIAiqfX$*^eyU`>VcTcJdp9yZfjcUI>>R0UpHD#`qh)Ip z&r`?#=xBCx*OUJTEsBTS)sagorEyJH!XA9F)l9C6`i1?+CkcX6^V-`J7;zt`jOQY| zvic9w0i})nTv+HL#GZsZ_;T zyo~GaXDJG`f6}tp92-a0ra3g& zah_5-H}4~B37z56-4nVuwr3;%?l%GyG97j?OTL`-`eWspkrL6k76O7Mg+Vi#v4;!Qm+TGK&CSRS%5!2afAAi6Gu~cj- z=DCcd>dHNfI@Rm;DK+qUeWvp@dRf$19t5<4N7Cy^kJGBCwI;Z1PF68d>_9H!luWTh zTug=zgg_S?t4xJw(HFaQAO}s9NkB~Ma&48$gOPXlrnkgyyMsT~vNBJ(3%3QznT&5f zT6F8928byW6CS-3uCRuDHeWMP?B8ly{kwLXBu5S|FW7VWYupb=pSJ_-8wlUrBfmSK zG3Rj#?l%*txZItjuR*-8&K_%)RpN&adHbIm8Uu^sI##ZGgQuvZm0B}-7%$jSu#KRB zJ)scIzBCPr26QUDtGMunQ6C<~w$zEiShny#E79-o=lqaHdwMOmqHVWR!>c|nMe{#n zkF@46caxYt;(XKBI}OJCn5jT84;5a=YD6p+ys_vyVaPP~#K0nlGtQu!?d=M9nvsVs z=i7akgDrN_*<_o&ZBY?3(p*%Zw?D^z*cB3T?B{onWOQr&3Y-Z==$;OeoOlmmS)kD= zc9}S}%kv?BoAX&bmKISU$13hH`Cd{dwlpdEx9e#Sa!$~_tFn_|HdQvn=S>sG*tANF z6?vat@B8doulN^euDrpDGe(6T`Wb)r1}^Mr$K_J9Lf#{mh>(pQK-P?YP~~1uU$@?! z;pw9D=XGarSYG1<|K%y8mN|NMiPOl_Mk13UcsCYyR>?M<@ep&=wB&j1#W7GsQ9Z5Y zdF}jLJO0rGevA`F}I;Y{+);T6(#p>lp={KLBHKv#VAXPLu++Y%3PZxKH7v|)v; zhDQ#|k|sKl9|0vGo-j=;0XL|yowRy=ZT6`{(gt)3d}{2=;sqxY)ir=rUVHu0rDZmf z3{_!T`oZ%n?plDfb-3H-!!3dqofVf+Fo4Lyv@;EnyrABiqP8?B^)r6QNs7eWokI$n zp~jz#<0o;dz+2()2%&#Bj5D}XXye#}_|^VR6XCzOd*R2h4J)=< zTQSxacW94b_$rNzpH;#bKA1%~Irx)XIsbyr@IV@!2q8HF^;?dzs{WPV!$O-Qj2D$!Ngnk$<`*!(Krom*F)ACQ+O^r7}Q^32OZ< z;(TJ9X2Yn79mCGo!7fws0?)xS41a7{g3#sbx&xwm_GE9H@P7JK>?s_Y4EhiqU^ahK zGvIdi5?Cy082D3^MHHL=1#n8|eFbsJGuAjd`7vy~M7r3UgErUvE$X4t3V9+;vGF&H z9owqdNgW+S(Cw^~C1zWkZI%o6IfwOd+zsQ?ar-xs)1LHBOIBU-mK1LJx?7w{<`qOe zztw`!DIne;6_QMLM6A5*Ug5{`#I*wCNPV&e#^W@5LFv+j6}8-do&5Ts;uX+cb8d=Z?Oi=DRjjS$y;6oE^bmA?%vU275cHw8Ws`U7%JT{R@Azalu}DCz%`KUd_#?w z9It!K^v|(NZo@3YI0$Cu(Q`d{@D}zAeI4365S7K@vScQT|I8^E%xe98R@fu?iG|~s zYjHVq{e>j7OxnZD_{8y9N6&rGeEdRP88W*uiF*TEy%lL%{(c7Q3=Zg%q+lIqM$k2Z z2P%B}_pko@$>OZ0StfE^ofeiK)mqo@JnUl%zl>(9B|#8;jUuaqMwlA!`N?FO{-hIs z*l2_x6xzFBpG9}C@R0mSn!Y81$*|_(Iwfxmm@C-&^7%z*tdplt7E>Yp5|&nBLLlsl z>Emd+wKdJx?Nuf$yJp7@>-nlC%!cD|V~e<1h;RY}t`LYYo398A`Ip42Tk|w_;Y^{I ze>xK;>I#@swA+u*rZ(AH5hewBkq9-b^)@LF5jVd5Jsdl0RH_SQVVI@nTjD||vCY1b zO;szzqiQP_#@_rQ()|zik&V}(@ZUx-mPvGieh>J4{vmyN0z>?$Q?GIME&Ee>doXK` zoC}`dBlKba6mL;2w~&omy(@!2Y#1|%JR88_B1+%iQu*(z^UsDwWk%&4DECYEJ3hU- zx}Zdl)6_?nmHHeOKDv8FzM@D*8WP(ala`e9OfyrXqXlX{1;2MlW^QY%MjHdIWo|q7 z$`f1&ea@Abx-%kTFEsInz9?Q?Mt6Y=UiA;3CY_X_YhbECEX)In*#7o9d$*R|1oLZ_ zDNKgKA+>iNmcMsb0QA=^lu%Hr!V$R#X6T8MUs5a*C195gxcG*r!$eAW|K-SQb2(YZ{_qEHn~ZB9WSuxPd;L zKbmJ~yX>(an-khu6G6-Me)88O|COB06l>Xeb-W=m8~9ANR8z!06Ps^&#yb_6N_FQ% zUtk@P)(}=q>SmqJ41$}JB8Fg;5G-IV()-dHEpRrzRwi(b<@c#@kex$dL&o z8AnW+8eSFH09A&e#ZlGMVPL5o0BHMY*;!tu(n^(O*K-{`O^miIWZ4;1Yx|#EAlfVws0>*wj@|k6PRp zDVP4oNOGJkwm6{*_1k$rbhi1JzBwNm_^d*K#fM8gof{!!ezB)6$d$wsQ&bK*d^ zPu8!IF9)CrrWAC%cZw)M!@!#bQ{2`A zq>uNm4{@mme+V4YG!>k%UOi(@(|z=36$@xPd||c@Ty0)l$jA5EM^kC!GJUsUbX}_J zZR36{?$9R*-9DpVC3#uKX7%v8xyc+xSvC3-%Onq%$2>4>~} zylK8FF^#GpkRc#${xMa8G5;ccl}feQpk>F#Hu>>kM+GCTYgL0nkT(YLHXB?Ms=Ca2 zSC=|XI-Rlnet9W)0btKTN;Ru6*H!5j6~b}c*%kyI(tINm#AI53Lm6`6QU!h)XQin7 zJ9yeIC-e%2)ac&4+Kfy=Z?Nl?eWSza*HKkf3#-Hm#`B}7m$uVs7lGH^U?F=u|UB7eaD_E;P+(KUX^h9$xBjBuwz_W@mXMIM_w zD~X~>R%G6HNc+NmvVHIJ%f#D(XTM_h6uiFeL@|HgO|ZE~P$!=K2`V?|JvWDOD6V#1 zn*=92%r(k$B3%FADN_z`NcuB@urlYnphH%n%2~>D3^oq*7#BwnXh}1E*||#rsa1lu zhy8DNb$Hkdx;sQ)1x46z9vbYVOIET;i4wa>IeZ*ohGq5RLpceju&Y+-{<$TJ!c@U6Wk8iA;NzulpaT%4PbMpB+>Q%G&nqzIx69 z$89@rfgw{@1mgxQd(7;3g`pZLHxaaw{@wb3rmE1T@O&!vrYs^};!G}YR&i8ae&j_# zh-5w@AeHsDn0k1r^4VyCPbu~6l%}lKiOZOqy5O43FzwH;~|H7AqkK%J&1W*t@hbW{_i6muGRnFfd$m6t}T?puZQX=ou zIR4?H(USahZ8wFWOZ`T)_d3Vk*PPon%^jyH$6H!AvbrCJ z|1wQ`;uOa12`WEw-W~&dXx&^wN_Rkfylc?p#vtAq0!GQRU*z^SZ4Aw?iwe%v zGN{)JbtA(-;SNUOC!-x3W<1@+(eC*?IrUX-H(dTuj_WS)LVW|d;N9^D7HM`9)+K&u zwlG9RL9HImppFgiG%y)f8DkPbIfW|}1EMfAA>^HaVGL+G#X=2chd=fiNMy z)mn}s@D-fV`pm93tQY@uh&hA4+Tv^w9ybX6Sifa#^Sar|`lL-_afhFWcSje2QoJ7i zFH=jGvvsgK_Q`6c3zvKe-lh>7URi}RtnA9{0LFvfR~9rz`<1P@Y^+^%%$z??@F#$M zX$@(|4)@U&Dm!U@H2P;a)#Zu|Qy_~BdFQZex+O+=5&~OQF$nJ)M%P&s+uyqu80Y4b z{l{jW{VenjtbT=);-vRD-=@G#kaDGZ#QQD#DLE&b#kTzhP=nS+10M21&(0R~rG9bd za}ej=4+ySYWi`KY{bVG|YmJmk(%V50M6bo)h2az(oIDwtc5su@>YzZ#97&=k2iXX9 z9IoB73Me3nWmju8Q5OZ81M9fPncyip;eo#hXO-F#pD2$MbC?CK^QFivgibG7W$_ePh!=9QqZDI?8 zfdD#gx>Y&x2M~!Pm%hqv>g!#rE_w7AouVHke|;8nBmVlP)n~z$a6IE3@o&>>x^aHy zZ{v!Wvk!vNcC+g2rK8S3AN*`zQh^$=#V$$=>~`7s$xP@G6zXVWqi_^tlmY5^#Ce`D z=gB5GY}sb8Z31nN!bh%tA2k0u*}~3Og1)A+{&6!L)Larb;Z7Oi#VPkU!%{O_!8q*D zAr-iOe)L)N^#}!{j|W)mx#C-j>-5aL`|Ka0oj%>cnktvya;a*5FXg*3SJ7g${?w&q zXq@^KwTFA?g;~3EiqY3YL(O;EMGjcG{*wETS+JSz%RevAR2`Ho8wKa9*T~aqu9~^} z*iU-G{gMrqX^eSM4TLhNsExgn9@aRomm7P;D(UGJ3%(fbM&xVk2SH9ZYd?nrkq4QaLiD%52P zUSe_Z&tg?4X%5Dek~{f9%7&!hGSZ<|B9aLu42CY~6uTY37W52;WE@Mq@#LYj5e&`# z$pi*6Dc8%aD|ZzC3Bl|Paj~ucKo@U6(CZe+yBpl-w&mfSTaOH@DhSc`NZ{syvG=?l zp@DmtMh}*<4Ib7#5&eq!uEq>8Q+LhO30?ChawACH-1h{iLKV6HeR$TOZ?(5OvP9G} ziuq^{2KcL|l`&LeRS*7%8-ykkjkv~R$v*Zt^7FAx)i{53;2D3V(;p+n78*v%Ekucg zW|TQ`{Utar*U)AAT6b8c;_qQav(SMin#}DI`86h4ba;P!5m;ac3U27Jut)1EpufuY z^y)3Hu%0-Q7xy%)F6#W;ttcpRM*sM$FAM?GR?SZ=K!vI)Sr7NeXcVd20w%GBOgIYa zn6xq|6ojIZt#qqs$}K(AL3zJ`9iu6<)p}Xp`aUzt@uhk>!?4^P&!S#@H9&zJWiIOH zxn6vF3Tjt$h@Y^@$77QxXaGm_{aX)!|l47ZGgS-VUI7Ox7c`eXz66cmpgsE>7S{y7Q%DVJwUh1hW|5vikKuVn2S580V)V5|8rsd$1A3xP24o4P?usN(&b zL+n<8#sVRD{THG%g5NmYrywFmY@;G^8c9(#!$bSy9?LV1f_Fd%!o5()a#Nzj)(tk(#Ihh-TjEF$DSbjZdcGTW{Nm zdmtL6ln0U_8qxGy;nz8DHRGu~{O!DyyMtgQ?(JbZ%jJi|3*6XWRW9bf;FljRcDM8v zIYWU-e1T~VN!L9*vMvIi6}W$Gz()osCJ#rgrrQSoPTM8F-%oB=&ep$O99PF}&(8<% zQ*8wBI%vIWU}1huDE33=d`6CD!Vvvx2s_x%M^w=N-HY4R?kUIL$*Q&~SYjU?Q$PKa z{94L>huhPIW&W%TeQQc7}3C|2jafJi>Qk*uPV#ZI6;UO z^$P`v64~~BMR7h%5HO;LkSIZqVy&25$5xp~*5*0}J~Va>bp3C6j}fb*QWt@3!cobp!nW!%hB!>MaRT<~t_Q*qWuJ5weq{<5zMS94U*&m*$SirKKzJ zi$#>ik>&=Kja-F0be$%|u zo?G@?cRqi&AvyL%MX(qd`L#JVIDbmd!aL|AMQpz0s(%eS3ru{^$T3`DjbWk5BIOH& zy_tP<)V5ol*qr}|9`7o}jw7-@E&{&d8|^zEk}rAZ#XfIdFJvjwOqc7V&SM1liS#+l zryV~@M?G1zvo_{(oaitF6o@={G(R=~y#ES{IdX@T0M>_7xI2OCgPF5b0`+rRS8z5f zW<`Ye48*22bx#>zq0uNPqeJvW!w^%$QNdeOXMl5S={{~^Mps(2N~Q%lgaNV5KVffmM@%#CW%~{#IgI5&nBT%J>g8qC4bV<*7@OpV{Qc$+i6d~)cXsT zd`BecUuO&YQQ4IaxtEn*_Qm>-d?3Q>eKwAnM8_H3nC*`39-?b=iSa~8+02YIL*haO zLB#$PhlPhuLzV}I??KdgBW5_z?hB|Hyo;0%^H;V7dykqUqr-*0A}JX5G2yelUd@4!{tI)1m6QwdziF%uYjk zrO#+7v^xuDoxT`aTvQdiWW~$-G=WVf=vU)+oxdGCBtttnJ%4eNPcM4h0tdZYaxQ0l zXRf*#$zk&IL;CsW@u_(4(DL3Jh&PjTJi;gH)3iOgMWSa7eh_FV2J6z8#dVH}R! zy;I4i7+VRm`rC-|>#goq?|JoV_FeFcKkckio|I41h8f2?iH?n8uo8lyWlE%J5P%z7 zS!gS@onnDMt6#TnhHfeKgc#|h0g8EoCsym~+mtpY(Qt8v5@7A_y=9W~4+q~DtRRx7 z0V;ixI_EX@r}#_**Vz5QD7*FQ3M~*a;v@PHt5Jc>1O?knnPJs0#IU2>6u(aqK`q<& zQWww9<~4M{lO>E_(d=4fvjc#eeo!bANm zW`;|5^<24MpQR)?*83ukaIMix3#_por|$#eu;V0i zB>a~Zdm#l7xjxcrdf-W0F+=vho`Aw_S~yA>CEd`4m2zmkSm2VJ2MHN}jx=${(w( zbrvH>ToDT&Q|*nqc*VrK6b z8dqs8p4Z$U4Q+@(E^a9U-%-vc_~x&x9&wKG0^B~C5_oPg_xUano`g>2oJ!5p!(!(= zg+WybWX|1EA{CQ^lLegBXz`x78Y!*GGhn<<} zXh|4Tv4tz|IL{Dw^t{qXp9g^;y(c^*Ja>%6Kh-deayG7c4(d`>#JcthPplKC7iY##tmeN;9q^ zJ5qeXm!uKjteluMN5h{i-E<)-mT5N@%A`V$mBUzB;F7BAH4HiNwoo<&=VCE9A6yx}6!QNL$n!yfMv2H)^svdi9i>Z8(5OS-uF*Qr=ys$}d>8(o@~p zdtre{?XS1>WQuG*&V^)sG*n*G#v&#Y)q`cvi559@>_!$R-pb=2?@IPBpMRv`MOn6U z^2)Ja^&!k(m_DLrC^^m9RH}+oQ>I=o58tizND9-P+ZqVoW>;tV<7IzkwvP3{wU7vZ zX;-t@&3B|lpJALiaoP1<19O#`jDuA01s;3&Q*BR7)5Db9U#fI}QUiM}@yYD`?PZ>S zr-?clb+z3H`suAGBF;S{AQh_AP4>s^!4qH82gW0~4l#9~#B#EmT2US$uiMqLlJ3Ch^&Es3R1{{ zv=L=y*v~U@-KM_EEVqN+o&XpO6txa76PP!SG-M3P)BwJ+L_(G*mGU!D=jHsKbm~Em zU+i(R3?%7uA`n5-l(Z8eOSj&Z%}U(qP3*5K{e*NMSy>?wI%l;Tc3Xiw?jjs9Q~;V- z{&%;9Y$gZGb>+daN!$l#OQofXI$l#2>vVO`$3f+_X@b)=gzx2Pdy*2~M8m|)G@akU zMK=CCPByLIkf!;NW4VHlF=c$pw0=0X`JMbc*kwiRb)SA87v%BO2I_kz9v*(%CS zW31V}O*$x#o_Y&sNpclmBJxa*9PL&HoiC&fueVU!cDjIUXcXl&A++PbBKl)>i7 z78t|vAN&PgBlk<5V`fMSb2t*J#7bq455Ef1)c0{}uKU-7;K(TApFAd`4h9}sI1U$> z*f$ARc4+)Jdas}^z%GjC{R-}lg%>wlKMgHCi1yDODAuOSHbmWz$UJHOHI?^#olNCUn}Dj0Fy~x z;NigIb-|-qe_`vbL$jxhQsGiPS!1bOx*OyO_A=nC1e4SGl6a_)jeBAvLBucyhpeSO z0u`f+l6px7y+1Ntrw1tiBdlY`lsKBy?rV{jGDhCxCrj=yaHDEahVtqPTD6P39Nnky zVc7{cGh~Q;n%$|2dI+mpS&KBfoKD z6=grY+kig^?MqZR>Tt7^{Q2v%+v&|=2{s=8DUr6;4cEoX^KX~?keE=`pERnoE$*gm zzb^QM$CPN9_M)UC9awauZdsd7VUjV|`=O^+j1c*`2EN9$5xz8NokIe^mn?4|ucZ3- zVjN3sW|#W$nbq~Yry7HO0=UP^XU&gC4y+!(91a<7>S;6nMY7IxfC{txLdpTW1-yIy z#tXdk<@5fCr)mGhf4t_kRlBkH}hu!BR>~S ze`HKG3xmK}E+IJMk?)OhK}zm{C8WpV&{ztT0|w4}{mhB}#s7|SRa%R?aqtdW6&+kx zi6NKER_Vu2--ZNYI<^zi-R?&H!ks;8PmSXLy}|Y?~EGY z5x<~P@o%(XwE+oY_TaII?AG@mj1prbBu&M(shX+Zxh4SwJn~w3{6RPi66z$R&FU9F_9Uo6-QPXEABJ~o7DULD#Q5bOxKGbw*x0#N#I&5igwAape#)yTT@e*E?BC*2Q1 zEpb0p<4e3{@(`x3B)3*wV>*~~Z)-ihCAEb!d{qiN-th@}qVdhR5b(Ryc)4l=D0;j9 z#6wBz)}*}=x6r){i=x%4XvHAELHT%IVay%+Op*D20+ZNvD zM)?5FoXZxn7Vhe{zSQ;eeNTID!A8p?>v1~Ma#&t&5YWfX6g7s0B5yQds52!xh8`t8 z%6<3>WUuQT6GOtyA98qKO8V`6b~g?~Q)jH$2GvK^JZjeQfZe?M4uM<%iUfoqv87t7 zqmc#iQF6)3o%fYtQxRqyk5GE@b;WzH)8c;Kvo65*)(@L1tigFK*R>{pBXZ|8%~(_7 z51Qi%V=Rm4pud9nD7iE)rB8JT&w!(ePAxTeqX*3X)<#Pca*O>iRU)&bwkiU)t+Ll) zFz0Lr564Y={mnSLaK8ls=VRpHU`fb^u)IwMSK;?#sB)tW#%uY@O1{ z*!Ml_LY7gn_I97usNs8@ZJHJcW-|H)>xS+?TUo*|nNM^yGax?7yj>*UbOeYpAWo5; zo4+A*!0c~bfGu~0h52@9mT)dTY8!ag4wY^Ze&>9vW6G7ID#SQPd&LkHcIklW+set&>1=;1l#?{bduF=A zHh=x4%hmia>N~=~EUTC5_{e*|)JMtDD8oc<_QE`jvs(b-!M%Sjj3Hy+?R%^kr=!uz z8^9s+Xtp(d6K4EO7;;V${jEK^;n0{$4;I0h)9M8;PF?Qz>#4gSF{SB`47GzPCkxXq zr+DG|I#wyilM9Dq3FB5TqC=!1@lo>Ic_C+QFtY%Ta79zy_`md8Zhig!q2j_u zGF8TR$k%#8f?nwU%+Xu$@ifA~784x>3dBd5r|u%gV!B-kDrV0GTBu;a>~8@%o`?2N zaYnw?@Xa%CYz3i5spzFo7$#W|24^{<MO+S(M01paZSVBEb-JaS z&*SsSf@elW_GmiPB``KM7@C5^4s7~19>aGUFBN=o4F23k82@;!gS#s6;R_bI?#(=g zyET+{+$uqd&n;&Cy7M1>-{yJqT08lRA?+cpwq2Kt?OMT3vs?GU2;0J`RmKSzY(p5- zEr^b`w82szR9xLBz3xpu<6cQ<&l1bAny;BZA?5wwhjbSzdic4|^kh!#7CKj}xtG9a^UR1n#n<|Le@p76{}D+*3Yzk@twZlo)E|E9NmDg-U3Wbe_fD4v~pg#-p5EihYs6&UXXsV0lA(cjCTkdDV zA9Mugm{L7|{7!Q2`SRVI?|e78HN%`2i~}j~I+ycnNbe!MbBQn%cMu&1l>&aHxd4o2 zfR~*~xu+BZzd%ucwYnN-K7U+rA#+6U{r)eQhuj);P2LFynucGux6i~z#ZVsH5<}Km z!pM{m9V~5#PcN4@V}9A-6m$a7t06&_cm9dx_N2h{-IXh2L}KJ%#FRWN7Tj0zXvP~o zM4$B>`$SwjkrhG!_1Skx$2uO{dH+`|BJC9MmQOb?PjH)H7B!?-Wa zVC3vb>$Bd!PyE?}wHfIU`EtpCja0PY@98Qs=kGgMX?`jq;WrLKt#;2hXOWgrXdcbgJBheB1#%Z74zVTDHTXoWIh*q9arSO{>vB>L3Mo{lNvU$De2B>=j20}SgJv`F z>E$91?RQl-x8u!%Qc+&xC44m47t1E%P;;vzzFc{2uW^!>cTM-E@AcXyrf?uNNF{XBr2uP#J7){}$3uxw2#L zbJ26l^T|nDcfdek$mqx-5gxZD47Qa-M|CZS_UYx~>T>!>J~Ts7MQ`DAV`T<+PbdDO zG5pZy?Xgd^{YD@Qech?T%_<=*>*vHCW*tPg!&);Fs}kToV2n2krFCfei1={yjUVWB z)B7~By}4iPq|kL;k}>0G(KFMxlA0(-Kd(5r=3NLk2h!1rZ7R!K@*@a538T;3M28FM z#HZJ%vXdt}n|PgA{+12Hbi3H#`ZgE(XY?_+jn4{j+XQB*#%A)26tq{%+kzaWko`$jz?`< z>FVKkvS8)i@Tx;b5{&qRFtm3N9XEbVeE;xY|E;W`qZeRM;}H^wPt85;nbEVq0ehAP EV@ZX$kN^Mx literal 0 HcmV?d00001 From 165fa2e306594c3f05689ff8489fe6085bdcfbf1 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sun, 29 Apr 2018 18:49:32 +0100 Subject: [PATCH 5/5] Bug fixes with multi-threaded sequence capture and recording. Fix with multi-hypothesis visualization in matlab. --- .../Script_CECLM_cross_data.m | 21 ++-- .../Script_CECLM_menpo_test_frontal.m | 23 +--- .../Script_CECLM_menpo_test_profile.m | 25 ++--- .../Script_CECLM_menpo_valid.m | 22 +--- .../Script_CLNF_cross_data_general.m | 101 +++--------------- 5 files changed, 36 insertions(+), 156 deletions(-) diff --git a/matlab_version/experiments_menpo/Script_CECLM_cross_data.m b/matlab_version/experiments_menpo/Script_CECLM_cross_data.m index f4bd57d3..b846eeda 100644 --- a/matlab_version/experiments_menpo/Script_CECLM_cross_data.m +++ b/matlab_version/experiments_menpo/Script_CECLM_cross_data.m @@ -2,11 +2,13 @@ function Script_CECLM_cross_data() addpath(genpath('../')); -[images, detections, labels] = Collect_menpo_imgs('D:\Datasets\menpo/'); +[images, detections, labels] = Collect_menpo_imgs('G:\Datasets\menpo/'); %% loading the CE-CLM model and parameters [patches, pdm, clmParams, early_term_params] = Load_CECLM_general(); - +views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; +views = views * pi/180; + % As early termination weights were trained on part of menpo turn them off, % to perform a clean cross-data experiment early_term_params.weights_scale(:) = 1; @@ -53,18 +55,9 @@ for i=1:numel(images) bbox = squeeze(detections(i,:)); - % have a multi-view version - if(multi_view) - - views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; - views = views * pi/180; - - [shape,~,~,lhood,lmark_lhood,view_used] =... - Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); - - else - [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); - end + % The actual work get's done here + [shape,~,~,lhood,lmark_lhood,view_used] =... + Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); all_lmark_lhoods(:,i) = lmark_lhood; all_views_used(i) = view_used; diff --git a/matlab_version/experiments_menpo/Script_CECLM_menpo_test_frontal.m b/matlab_version/experiments_menpo/Script_CECLM_menpo_test_frontal.m index 6dab83a5..f1f6f6c5 100644 --- a/matlab_version/experiments_menpo/Script_CECLM_menpo_test_frontal.m +++ b/matlab_version/experiments_menpo/Script_CECLM_menpo_test_frontal.m @@ -3,14 +3,12 @@ function Script_CECLM_menpo_test_frontal() addpath(genpath('../')); addpath(genpath('./menpo_challenge_helpers')); -[images, detections] = Collect_menpo_test_frontal('D:\Datasets\menpo\testset\semifrontal/'); +[images, detections] = Collect_menpo_test_frontal('G:\Datasets\menpo\testset\semifrontal/'); %% loading the CE-CLM model and parameters [patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo(); - -% Use the multi-hypothesis model, as bounding box tells nothing about -% orientation -multi_view = true; +views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; +views = views * pi/180; %% Setup recording experiment.params = clmParams; @@ -53,19 +51,8 @@ for i=1:numel(images) bbox = squeeze(detections(i,:)); - % have a multi-view version - % have a multi-view version - if(multi_view) - - views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; - views = views * pi/180; - - [shape,~,~,lhood,lmark_lhood,view_used] =... - Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); - - else - [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); - end + [shape,~,~,lhood,lmark_lhood,view_used] =... + Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); shape = shape + 0.5; [~, name_org, ~] = fileparts(images(i).img); diff --git a/matlab_version/experiments_menpo/Script_CECLM_menpo_test_profile.m b/matlab_version/experiments_menpo/Script_CECLM_menpo_test_profile.m index ce5a7b2a..94817319 100644 --- a/matlab_version/experiments_menpo/Script_CECLM_menpo_test_profile.m +++ b/matlab_version/experiments_menpo/Script_CECLM_menpo_test_profile.m @@ -3,14 +3,12 @@ function Script_CECLM_menpo_test_profile() addpath(genpath('../')); addpath(genpath('./menpo_challenge_helpers')); -[images, detections] = Collect_menpo_test_profile('D:\Datasets\menpo\testset\profile/'); +[images, detections] = Collect_menpo_test_profile('G:\Datasets\menpo\testset\profile/'); %% loading the CE-CLM model and parameters [patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo(); - -% Use the multi-hypothesis model, as bounding box tells nothing about -% orientation -multi_view = true; +views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; +views = views * pi/180; %% Setup recording experiment.params = clmParams; @@ -41,7 +39,7 @@ if(~exist(out_pts, 'dir')) mkdir(out_pts); end -load('../pdm_generation/menpo_pdm/menpo_chin/conversion.mat'); +load('menpo_challenge_helpers/conversion.mat'); %% for i=1:numel(images) @@ -55,19 +53,8 @@ for i=1:numel(images) bbox = squeeze(detections(i,:)); - % have a multi-view version - % have a multi-view version - if(multi_view) - - views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; - views = views * pi/180; - - [shape,g_params,~,lhood,lmark_lhood,view_used] =... - Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); - - else - [shape,g_params,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); - end + [shape,g_params,~,lhood,lmark_lhood,view_used] =... + Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); shape = shape + 0.5; diff --git a/matlab_version/experiments_menpo/Script_CECLM_menpo_valid.m b/matlab_version/experiments_menpo/Script_CECLM_menpo_valid.m index 41bdca4c..a021dc19 100644 --- a/matlab_version/experiments_menpo/Script_CECLM_menpo_valid.m +++ b/matlab_version/experiments_menpo/Script_CECLM_menpo_valid.m @@ -2,14 +2,12 @@ function Script_CECLM_menpo_valid() addpath(genpath('../')); -[images, detections, labels] = Collect_valid_imgs('D:\Datasets\menpo/'); +[images, detections, labels] = Collect_valid_imgs('G:\Datasets\menpo/'); %% loading the CE-CLM model and parameters [patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo(); - -% Use the multi-hypothesis model, as bounding box tells nothing about -% orientation -multi_view = true; +views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; +views = views * pi/180; %% Setup recording experiment.params = clmParams; @@ -47,18 +45,8 @@ for i=1:numel(images) bbox = squeeze(detections(i,:)); - % have a multi-view version - if(multi_view) - - views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; - views = views * pi/180; - - [shape,~,~,lhood,lmark_lhood,view_used] =... - Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); - - else - [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); - end + [shape,~,~,lhood,lmark_lhood,view_used] =... + Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); all_lmark_lhoods(:,i) = lmark_lhood; all_views_used(i) = view_used; diff --git a/matlab_version/experiments_menpo/Script_CLNF_cross_data_general.m b/matlab_version/experiments_menpo/Script_CLNF_cross_data_general.m index ad4d2251..e1896039 100644 --- a/matlab_version/experiments_menpo/Script_CLNF_cross_data_general.m +++ b/matlab_version/experiments_menpo/Script_CLNF_cross_data_general.m @@ -3,51 +3,24 @@ function Script_CLNF_cross_data_general() addpath(genpath('../')); % Replace this with the location of the Menpo dataset -[images, detections, labels] = Collect_menpo_imgs('D:\Datasets\menpo/'); +[images, detections, labels] = Collect_menpo_imgs('G:\Datasets\menpo/'); %% loading the patch experts -clmParams = struct; +[ patches, pdm, clmParams ] = Load_CLNF_general(); +views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; +views = views * pi/180; -clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; -clmParams.numPatchIters = size(clmParams.window_size,1); +[ patches_51, pdm_51, clmParams_51, inds_full, inds_inner ] = Load_CLNF_inner(); -[patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams); - -%% Fitting the model to the provided image - -% the default PDM to use -pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; - -load(pdmLoc); - -pdm = struct; -pdm.M = double(M); -pdm.E = double(E); -pdm.V = double(V); - -clmParams.regFactor = [35, 27, 20, 20]; -clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; -clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5]; - -clmParams.startScale = 1; -clmParams.num_RLMS_iter = 10; -clmParams.fTol = 0.01; -clmParams.useMultiScale = true; -clmParams.use_multi_modal = 1; -clmParams.multi_modal_types = patches(1).multi_modal_types; - -% Loading the final scale -[clmParams_inner, pdm_inner] = Load_CLM_params_inner(); -clmParams_inner.window_size = [17,17;19,19;21,21;23,23]; -inds_inner = 18:68; -[patches_inner] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*general_no_out.mat', [], [], clmParams_inner); -clmParams_inner.multi_modal_types = patches_inner(1).multi_modal_types; +shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +lhoods = zeros(numel(images),1); % for recording purposes experiment.params = clmParams; -num_points = numel(M)/3; +num_points = numel(pdm.M)/3; shapes_all = cell(numel(images), 1); labels_all = cell(numel(images), 1); @@ -55,9 +28,6 @@ lhoods = zeros(numel(images),1); all_lmark_lhoods = zeros(num_points, numel(images)); all_views_used = zeros(numel(images),1); -% Use the multi-hypothesis model, as bounding box tells nothing about -% orientation -multi_view = true; verbose = false; % set to true to visualise the fitting tic @@ -72,55 +42,10 @@ for i=1:numel(images) bbox = detections(i,:); - % have a multi-view version - if(multi_view) - - views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; - views = views * pi/180; - - shapes = zeros(num_points, 2, size(views,1)); - ls = zeros(size(views,1),1); - lmark_lhoods = zeros(num_points,size(views,1)); - views_used = zeros(size(views,1),1); - - % Find the best orientation - for v = 1:size(views,1) - [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:)); - end - - [lhood, v_ind] = max(ls); - lmark_lhood = lmark_lhoods(:,v_ind); - - shape = shapes(:,:,v_ind); - view_used = views_used(v_ind); - - else - [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); - end - - % Perform inner face fitting - shape_inner = shape(inds_inner,:); - - [ a, R, T, ~, l_params, err] = fit_PDM_ortho_proj_to_2D_no_reg(pdm_inner.M, pdm_inner.E, pdm_inner.V, shape_inner); - if(a > 0.9) - g_param = [a; Rot2Euler(R)'; T]; - - bbox_2 = [min(shape_inner(:,1)), min(shape_inner(:,2)), max(shape_inner(:,1)), max(shape_inner(:,2))]; - - [shape_inner] = Fitting_from_bb(image, [], bbox_2, pdm_inner, patches_inner, clmParams_inner, 'gparam', g_param, 'lparam', l_params); - - % Now after detections incorporate the eyes back - % into the face model - - shape(inds_inner, :) = shape_inner; - - [ ~, ~, ~, ~, ~, ~, shape_fit] = fit_PDM_ortho_proj_to_2D_no_reg(pdm.M, pdm.E, pdm.V, shape); - - all_lmark_lhoods(:,i) = lmark_lhood; - all_views_used(i) = view_used; - - shape = shape_fit; - end + [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views); + + % Perform inner landmark fitting now + [shape, shape_inner] = Fitting_from_bb_hierarch(image, pdm, pdm_51, patches_51, clmParams_51, shape, inds_full, inds_inner); all_lmark_lhoods(:,i) = lmark_lhood; all_views_used(i) = view_used;