From 78b81e8a458496bab8eefa6202d1177fdac8b6e1 Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Wed, 24 Oct 2018 18:59:16 -0400 Subject: [PATCH] Add frame_step flag --- doc/demo_overview.md | 1 + doc/release_notes.md | 2 + examples/openpose/openpose.cpp | 4 +- .../1_custom_post_processing.cpp | 4 +- .../5_asynchronous_loop_custom_output.cpp | 4 +- .../6_synchronous_custom_postprocessing.cpp | 4 +- .../8_synchronous_custom_output.cpp | 4 +- include/openpose/flags.hpp | 2 + include/openpose/pose/enumClasses.hpp | 1 + .../openpose/pose/poseParametersRender.hpp | 10 ++-- include/openpose/producer/datumProducer.hpp | 30 ++++++++-- include/openpose/producer/enumClasses.hpp | 1 + include/openpose/wrapper/wrapperAuxiliary.hpp | 4 +- .../openpose/wrapper/wrapperStructInput.hpp | 19 ++++-- src/openpose/net/bodyPartConnectorBase.cpp | 6 +- src/openpose/pose/poseParameters.cpp | 59 ++++++++++++++++--- src/openpose/pose/poseParametersRender.cpp | 9 ++- src/openpose/pose/renderPose.cu | 30 +++++----- .../producer/imageDirectoryReader.cpp | 5 ++ src/openpose/producer/producer.cpp | 9 +++ src/openpose/producer/videoCaptureReader.cpp | 17 ++++++ src/openpose/utilities/flagsToOpenPose.cpp | 2 + src/openpose/wrapper/wrapperStructInput.cpp | 7 ++- 23 files changed, 177 insertions(+), 57 deletions(-) diff --git a/doc/demo_overview.md b/doc/demo_overview.md index c8a7d5a9..58647779 100644 --- a/doc/demo_overview.md +++ b/doc/demo_overview.md @@ -145,6 +145,7 @@ Each flag is divided into flag name, default value, and description. - DEFINE_int32(flir_camera_index, -1, "Select -1 (default) to run on all detected flir cameras at once. Otherwise, select the flir camera index to run, where 0 corresponds to the detected flir camera with the lowest serial number, and `n` to the `n`-th lowest serial number camera."); - DEFINE_string(ip_camera, "", "String with the IP camera URL. It supports protocols like RTSP and HTTP."); - DEFINE_uint64(frame_first, 0, "Start on desired frame number. Indexes are 0-based, i.e. the first frame has index 0."); +- DEFINE_uint64(frame_step, 1, "Step or gap between processed frames. E.g., `--frame_step 5` would read and process frames 0, 5, 10, etc.."); - DEFINE_uint64(frame_last, -1, "Finish on desired frame number. Select -1 to disable. Indexes are 0-based, e.g. if set to 10, it will process 11 frames (0-10)."); - DEFINE_bool(frame_flip, false, "Flip/mirror each frame (e.g. for real time webcam demonstrations)."); - DEFINE_int32(frame_rotate, 0, "Rotate each frame, 4 possible values: 0, 90, 180, 270."); diff --git a/doc/release_notes.md b/doc/release_notes.md index e4188aa9..84d33286 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -273,6 +273,8 @@ OpenPose Library - Release Notes 3. `tutorial_pose` and `tutorial_thread` renamed as `tutorial_developer`, not meant to be used by users, but rather for OpenPose developers. 7. Added a virtual destructor to almost all clases, so they can be inherited. Exceptions (for performance reasons): Array, Point, Rectangle, CvMatToOpOutput, OpOutputToCvMat. 8. Auxiliary classes in errorAndLog turned into namespaces (Profiler must be kept as class to allow static parameters). + 9. Added flag `--frame_step` to allow the user to select the step or gap between processed frames. E.g., `--frame_step 5` would read and process frames 0, 5, 10, etc. + 10. Added sanity checks to avoid `--frame_last` to be smaller than `--frame_first` or higher than the number of total frames. 2. Functions or parameters renamed: 1. By default, python example `tutorial_developer/python_2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368. 2. WrapperStructPose default parameters changed to match those of the OpenPose demo binary. diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index 5df23a08..217190fa 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -89,8 +89,8 @@ int openPoseDemo() opWrapper.configure(wrapperStructExtra); // Producer (use default to disable any input) const op::WrapperStructInput wrapperStructInput{ - producerSharedPtr, FLAGS_frame_first, FLAGS_frame_last, FLAGS_process_real_time, FLAGS_frame_flip, - FLAGS_frame_rotate, FLAGS_frames_repeat}; + producerSharedPtr, FLAGS_frame_first, FLAGS_frame_step, FLAGS_frame_last, FLAGS_process_real_time, + FLAGS_frame_flip, FLAGS_frame_rotate, FLAGS_frames_repeat}; opWrapper.configure(wrapperStructInput); // Consumer (comment or use default argument to disable any output) const op::WrapperStructOutput wrapperStructOutput{ diff --git a/examples/tutorial_add_module/1_custom_post_processing.cpp b/examples/tutorial_add_module/1_custom_post_processing.cpp index b967fd54..5c2d13c1 100644 --- a/examples/tutorial_add_module/1_custom_post_processing.cpp +++ b/examples/tutorial_add_module/1_custom_post_processing.cpp @@ -102,8 +102,8 @@ int tutorialAddModule1() opWrapperT.configure(wrapperStructExtra); // Producer (use default to disable any input) const op::WrapperStructInput wrapperStructInput{ - producerSharedPtr, FLAGS_frame_first, FLAGS_frame_last, FLAGS_process_real_time, FLAGS_frame_flip, - FLAGS_frame_rotate, FLAGS_frames_repeat}; + producerSharedPtr, FLAGS_frame_first, FLAGS_frame_step, FLAGS_frame_last, FLAGS_process_real_time, + FLAGS_frame_flip, FLAGS_frame_rotate, FLAGS_frames_repeat}; opWrapperT.configure(wrapperStructInput); // Consumer (comment or use default argument to disable any output) const op::WrapperStructOutput wrapperStructOutput{ diff --git a/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp b/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp index a5382108..8e08c1a9 100644 --- a/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp +++ b/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp @@ -184,8 +184,8 @@ int tutorialApiCpp5() opWrapperT.configure(wrapperStructExtra); // Producer (use default to disable any input) const op::WrapperStructInput wrapperStructInput{ - producerSharedPtr, FLAGS_frame_first, FLAGS_frame_last, FLAGS_process_real_time, FLAGS_frame_flip, - FLAGS_frame_rotate, FLAGS_frames_repeat}; + producerSharedPtr, FLAGS_frame_first, FLAGS_frame_step, FLAGS_frame_last, FLAGS_process_real_time, + FLAGS_frame_flip, FLAGS_frame_rotate, FLAGS_frames_repeat}; opWrapperT.configure(wrapperStructInput); // Consumer (comment or use default argument to disable any output) const auto displayMode = op::DisplayMode::NoDisplay; diff --git a/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp b/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp index 047003e1..dd29f3eb 100644 --- a/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp +++ b/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp @@ -152,8 +152,8 @@ int tutorialApiCpp6() opWrapperT.configure(wrapperStructExtra); // Producer (use default to disable any input) const op::WrapperStructInput wrapperStructInput{ - producerSharedPtr, FLAGS_frame_first, FLAGS_frame_last, FLAGS_process_real_time, FLAGS_frame_flip, - FLAGS_frame_rotate, FLAGS_frames_repeat}; + producerSharedPtr, FLAGS_frame_first, FLAGS_frame_step, FLAGS_frame_last, FLAGS_process_real_time, + FLAGS_frame_flip, FLAGS_frame_rotate, FLAGS_frames_repeat}; opWrapperT.configure(wrapperStructInput); // Consumer (comment or use default argument to disable any output) const op::WrapperStructOutput wrapperStructOutput{ diff --git a/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp b/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp index 00e118e0..6e0f4504 100644 --- a/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp +++ b/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp @@ -199,8 +199,8 @@ int tutorialApiCpp8() opWrapperT.configure(wrapperStructExtra); // Producer (use default to disable any input) const op::WrapperStructInput wrapperStructInput{ - producerSharedPtr, FLAGS_frame_first, FLAGS_frame_last, FLAGS_process_real_time, FLAGS_frame_flip, - FLAGS_frame_rotate, FLAGS_frames_repeat}; + producerSharedPtr, FLAGS_frame_first, FLAGS_frame_step, FLAGS_frame_last, FLAGS_process_real_time, + FLAGS_frame_flip, FLAGS_frame_rotate, FLAGS_frames_repeat}; opWrapperT.configure(wrapperStructInput); // Consumer (comment or use default argument to disable any output) const auto displayMode = op::DisplayMode::NoDisplay; diff --git a/include/openpose/flags.hpp b/include/openpose/flags.hpp index d4592e86..e70a94f2 100644 --- a/include/openpose/flags.hpp +++ b/include/openpose/flags.hpp @@ -47,6 +47,8 @@ DEFINE_int32(flir_camera_index, -1, "Select -1 (default) to " serial number, and `n` to the `n`-th lowest serial number camera."); DEFINE_string(ip_camera, "", "String with the IP camera URL. It supports protocols like RTSP and HTTP."); DEFINE_uint64(frame_first, 0, "Start on desired frame number. Indexes are 0-based, i.e. the first frame has index 0."); +DEFINE_uint64(frame_step, 1, "Step or gap between processed frames. E.g., `--frame_step 5` would read and process frames" + " 0, 5, 10, etc.."); DEFINE_uint64(frame_last, -1, "Finish on desired frame number. Select -1 to disable. Indexes are 0-based, e.g. if set to" " 10, it will process 11 frames (0-10)."); DEFINE_bool(frame_flip, false, "Flip/mirror each frame (e.g. for real time webcam demonstrations)."); diff --git a/include/openpose/pose/enumClasses.hpp b/include/openpose/pose/enumClasses.hpp index 7712a7a0..71a7b50a 100644 --- a/include/openpose/pose/enumClasses.hpp +++ b/include/openpose/pose/enumClasses.hpp @@ -24,6 +24,7 @@ namespace op BODY_65, /**< Experimental. Do not use. */ CAR_12, /**< Experimental. Do not use. */ BODY_25D, /**< Experimental. Do not use. */ + BODY_23, /**< Experimental. Do not use. */ Size, }; diff --git a/include/openpose/pose/poseParametersRender.hpp b/include/openpose/pose/poseParametersRender.hpp index c5de7033..2c61092a 100644 --- a/include/openpose/pose/poseParametersRender.hpp +++ b/include/openpose/pose/poseParametersRender.hpp @@ -110,11 +110,11 @@ namespace op 170.f, 0.f, 255.f, \ 255.f, 0.f, 255.f, \ 85.f, 0.f, 255.f - // BODY_25E - #define POSE_BODY_25E_PAIRS_RENDER_GPU \ - 1,8, 1,2, 1,5, 2,3, 3,4, 5,6, 6,7, 8,9, 9,10, 10,11, 8,12, 12,13, 13,14, 1,0, 0,15, 15,17, 0,16, 16,18, 14,19,19,20,14,21, 11,22,22,23,11,24 - #define POSE_BODY_25E_SCALES_RENDER_GPU 1 - #define POSE_BODY_25E_COLORS_RENDER_GPU \ + // BODY_23 + #define POSE_BODY_23_PAIRS_RENDER_GPU \ + 1,7, 4,10, 0,1, 0,4, 1,2, 2,3, 4,5, 5,6, 7,8, 8,9, 10,11, 11,12, 0,13, 13,15, 0,14, 14,16, 12,17,17,18,12,19, 9,20,20,21,9,22 + #define POSE_BODY_23_SCALES_RENDER_GPU 1 + #define POSE_BODY_23_COLORS_RENDER_GPU \ 255.f, 0.f, 85.f, \ 255.f, 0.f, 0.f, \ 255.f, 85.f, 0.f, \ diff --git a/include/openpose/producer/datumProducer.hpp b/include/openpose/producer/datumProducer.hpp index fd124f2a..48dbd2ec 100644 --- a/include/openpose/producer/datumProducer.hpp +++ b/include/openpose/producer/datumProducer.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace op @@ -15,7 +16,7 @@ namespace op { public: explicit DatumProducer(const std::shared_ptr& producerSharedPtr, - const unsigned long long frameFirst = 0, + const unsigned long long frameFirst = 0, const unsigned long long frameStep = 1, const unsigned long long frameLast = std::numeric_limits::max(), const std::shared_ptr, std::atomic>>& videoSeekSharedPtr = nullptr); @@ -28,6 +29,7 @@ namespace op const unsigned long long mNumberFramesToProcess; std::shared_ptr spProducer; unsigned long long mGlobalCounter; + unsigned long long mFrameStep; unsigned int mNumberConsecutiveEmptyFrames; std::shared_ptr, std::atomic>> spVideoSeek; @@ -49,20 +51,40 @@ namespace op { template DatumProducer::DatumProducer(const std::shared_ptr& producerSharedPtr, - const unsigned long long frameFirst, const unsigned long long frameLast, + const unsigned long long frameFirst, const unsigned long long frameStep, + const unsigned long long frameLast, const std::shared_ptr, std::atomic>>& videoSeekSharedPtr) : mNumberFramesToProcess{(frameLast != std::numeric_limits::max() ? frameLast - frameFirst : frameLast)}, spProducer{producerSharedPtr}, mGlobalCounter{0ll}, + mFrameStep{frameStep}, mNumberConsecutiveEmptyFrames{0u}, spVideoSeek{videoSeekSharedPtr} { try { - if (spProducer->getType() != ProducerType::Webcam) + // Sanity check + if (frameLast < frameFirst) + error("The desired initial frame must be lower than the last one (flags `--frame_first` vs." + " `--frame_last`). Current: " + std::to_string(frameFirst) + " vs. " + std::to_string(frameLast) + + ".", __LINE__, __FUNCTION__, __FILE__); + if (frameLast != std::numeric_limits::max() + && frameLast > spProducer->get(CV_CAP_PROP_FRAME_COUNT)-1) + error("The desired last frame must be lower than the length of the video or the number of images." + " Current: " + std::to_string(frameLast) + " vs. " + + std::to_string(intRound(spProducer->get(CV_CAP_PROP_FRAME_COUNT))-1) + ".", + __LINE__, __FUNCTION__, __FILE__); + // Set frame first and step + if (spProducer->getType() != ProducerType::FlirCamera && spProducer->getType() != ProducerType::IPCamera + && spProducer->getType() != ProducerType::Webcam) + { + // Frame first spProducer->set(CV_CAP_PROP_POS_FRAMES, (double)frameFirst); + // Frame step + spProducer->set(ProducerProperty::FrameStep, (double)frameStep); + } } catch (const std::exception& e) { @@ -164,7 +186,7 @@ namespace op datums = nullptr; // Increase counter if successful image if (datums != nullptr) - mGlobalCounter++; + mGlobalCounter += mFrameStep; } } // Return result diff --git a/include/openpose/producer/enumClasses.hpp b/include/openpose/producer/enumClasses.hpp index a923d451..c56364ca 100644 --- a/include/openpose/producer/enumClasses.hpp +++ b/include/openpose/producer/enumClasses.hpp @@ -14,6 +14,7 @@ namespace op AutoRepeat = 0, Flip, Rotation, + FrameStep, Size, }; diff --git a/include/openpose/wrapper/wrapperAuxiliary.hpp b/include/openpose/wrapper/wrapperAuxiliary.hpp index f4ed032e..bb01b32a 100644 --- a/include/openpose/wrapper/wrapperAuxiliary.hpp +++ b/include/openpose/wrapper/wrapperAuxiliary.hpp @@ -217,8 +217,8 @@ namespace op if (oPProducer) { const auto datumProducer = std::make_shared>( - wrapperStructInput.producerSharedPtr, wrapperStructInput.frameFirst, wrapperStructInput.frameLast, - spVideoSeek + wrapperStructInput.producerSharedPtr, wrapperStructInput.frameFirst, wrapperStructInput.frameStep, + wrapperStructInput.frameLast, spVideoSeek ); datumProducerW = std::make_shared>(datumProducer); } diff --git a/include/openpose/wrapper/wrapperStructInput.hpp b/include/openpose/wrapper/wrapperStructInput.hpp index d2ded1e2..ee7c5dfb 100644 --- a/include/openpose/wrapper/wrapperStructInput.hpp +++ b/include/openpose/wrapper/wrapperStructInput.hpp @@ -1,6 +1,7 @@ #ifndef OPENPOSE_WRAPPER_WRAPPER_STRUCT_INPUT_HPP #define OPENPOSE_WRAPPER_WRAPPER_STRUCT_INPUT_HPP +#include // std::numeric_limits #include #include @@ -14,19 +15,26 @@ namespace op { /** * Producer which will generate the frames. - * Set to nullptr to disable the whole input, i.e. if the user is going to use his own frames generator. + * Set to nullptr to disable the whole input, i.e., if the user is going to use his own frames generator. */ std::shared_ptr producerSharedPtr; /** - * First image to obtain. + * First image to process. * Default: 0. */ unsigned long long frameFirst; /** - * Last image to obtain. - * Default: -1 (i.e. obtain all frames). + * Step or gap across processed frames. + * Default: 1 (i.e., process all frames). + * Example: A value of 5 would mean to process frames 0, 5, 10, etc. + */ + unsigned long long frameStep; + + /** + * Last image to process. + * Default: -1 (i.e., process all frames). */ unsigned long long frameLast; @@ -57,7 +65,8 @@ namespace op * Since all the elements of the struct are public, they can also be manually filled. */ WrapperStructInput(const std::shared_ptr producerSharedPtr = nullptr, - const unsigned long long frameFirst = 0, const unsigned long long frameLast = -1, + const unsigned long long frameFirst = 0, const unsigned long long frameStep = 1, + const unsigned long long frameLast = std::numeric_limits::max(), const bool realTimeProcessing = false, const bool frameFlip = false, const int frameRotate = 0, const bool framesRepeat = false); }; diff --git a/src/openpose/net/bodyPartConnectorBase.cpp b/src/openpose/net/bodyPartConnectorBase.cpp index 0aa38b83..c3d42bb6 100644 --- a/src/openpose/net/bodyPartConnectorBase.cpp +++ b/src/openpose/net/bodyPartConnectorBase.cpp @@ -44,7 +44,7 @@ namespace op count++; } } - if (count/(float)numberPointsInLine > interMinAboveThreshold) + if (count/T(numberPointsInLine) > interMinAboveThreshold) return sum/count; } return T(0); @@ -66,7 +66,7 @@ namespace op try { if (poseModel == PoseModel::BODY_25E) - error("BODY_25 not implemented for CPU body connector.", __LINE__, __FUNCTION__, __FILE__); + error("BODY_25E not implemented for CPU body connector.", __LINE__, __FUNCTION__, __FILE__); // std::vector, double>> refers to: // - std::vector: [body parts locations, #body parts found] @@ -694,7 +694,7 @@ namespace op poseKeypoints[baseOffset + 2] = peaksPtr[bodyPartIndex]; } } - poseScores[person] = personPair.second / (float)(numberBodyPartsAndPAFs); + poseScores[person] = personPair.second / T(numberBodyPartsAndPAFs); } } catch (const std::exception& e) diff --git a/src/openpose/pose/poseParameters.cpp b/src/openpose/pose/poseParameters.cpp index 567b0c65..20ec0e4f 100644 --- a/src/openpose/pose/poseParameters.cpp +++ b/src/openpose/pose/poseParameters.cpp @@ -93,6 +93,32 @@ namespace op {18, "LEar"}, {19, "Background"} }; + const std::map POSE_BODY_23_BODY_PARTS { + {0, "Nose"}, + {1, "RShoulder"}, + {2, "RElbow"}, + {3, "RWrist"}, + {4, "LShoulder"}, + {5, "LElbow"}, + {6, "LWrist"}, + {7, "RHip"}, + {8, "RKnee"}, + {9, "RAnkle"}, + {10, "LHip"}, + {11, "LKnee"}, + {12, "LAnkle"}, + {13, "REye"}, + {14, "LEye"}, + {15, "REar"}, + {16, "LEar"}, + {17, "LBigToe"}, + {18, "LSmallToe"}, + {19, "LHeel"}, + {20, "RBigToe"}, + {21, "RSmallToe"}, + {22, "RHeel"}, + {23, "Background"} + }; const std::map POSE_BODY_59_BODY_PARTS { // Body {0, "Nose"}, @@ -261,13 +287,20 @@ namespace op std::vector{ 0,1, 14,15, 22,23, 16,17, 18,19, 24,25, 26,27, 6,7, 2,3, 4,5, 8,9, 10,11, 12,13, 30,31, 32,33, 36,37, 34,35, 38,39, 20,21, 28,29, 40,41,42,43,44,45, 46,47,48,49,50,51 }, + // BODY_23 + std::vector{ + // Minimum spanning tree + 0,1, 2,3, 4,5, 6,7, 8,9, 10,11, 12,13, 14,15, 16,17, 18,19, 20,21, 22,23, 24,25, 26,27, 28,29, 30,31, 32,33, 34,35, 36,37, 38,39, 40,41, 42,43, + // Redundant ones + 44,45, 46,47, 48,49, 50,51, 52,53, 54,55, 56,57, 58,59, 60,61, 62,63, 64,65, 66,67, 68,69, 70,71, 72,73 + }, }; // POSE_BODY_PART_MAPPING on HPP crashes on Windows at dynamic initialization if it's on hpp const std::array, (int)PoseModel::Size> POSE_BODY_PART_MAPPING{ POSE_BODY_25_BODY_PARTS,POSE_COCO_BODY_PARTS, POSE_MPI_BODY_PARTS, POSE_MPI_BODY_PARTS, POSE_BODY_19_BODY_PARTS,POSE_BODY_19_BODY_PARTS,POSE_BODY_59_BODY_PARTS,POSE_BODY_19_BODY_PARTS, POSE_BODY_25_BODY_PARTS,POSE_BODY_25_BODY_PARTS,POSE_BODY_65_BODY_PARTS,POSE_CAR_12_PARTS, - POSE_BODY_25_BODY_PARTS + POSE_BODY_25_BODY_PARTS,POSE_BODY_23_BODY_PARTS }; const std::array POSE_PROTOTXT{ @@ -284,6 +317,7 @@ namespace op "pose/body_65/pose_deploy.prototxt", "car/car_12/pose_deploy.prototxt", "pose/body_25d/pose_deploy.prototxt", + "pose/body_23/pose_deploy.prototxt", }; const std::array POSE_TRAINED_MODEL{ "pose/body_25/pose_iter_584000.caffemodel", @@ -299,12 +333,13 @@ namespace op "pose/body_65/pose_iter_XXXXXX.caffemodel", "car/car_12/pose_iter_XXXXXX.caffemodel", "pose/body_25d/pose_iter_XXXXXX.caffemodel", + "pose/body_23/pose_iter_XXXXXX.caffemodel", }; // Constant Array Parameters // POSE_NUMBER_BODY_PARTS equivalent to size of std::map POSE_BODY_XX_BODY_PARTS - 1 (removing background) const std::array POSE_NUMBER_BODY_PARTS{ - 25, 18, 15, 15, 19, 19, 59, 19, 25, 25, 65, 12, 25 + 25, 18, 15, 15, 19, 19, 59, 19, 25, 25, 65, 12, 25, 23 }; const std::array, (int)PoseModel::Size> POSE_BODY_PART_PAIRS{ // BODY_25 @@ -365,6 +400,14 @@ namespace op std::vector{ 1,8, 1,2, 1,5, 2,3, 3,4, 5,6, 6,7, 8,9, 9,10, 10,11, 8,12, 12,13, 13,14, 1,0, 0,15, 15,17, 0,16, 16,18, 2,17, 5,18, 14,19,19,20,14,21, 11,22,22,23,11,24 }, + // BODY_23 + std::vector{ + // Minimum spanning tree + 0,1, 1,2, 2,3, 0,4, 4,5, 5,6, 7,8, 8,9, 10,11, 11,12, 0,13, 13,15, 0,14, 14,16, 12,17, 17,18, 12,19, 9,20, 20,21, 9,22, 1,7, 4,10, + // Redundant ones + // Ears-shoulders, ears, shoulders-wrists, hips-ankles, wrists, ankles, wrists-hips, small toes-ankles, hips) + 1,15, 4,16, 15,16, 1,3, 4,6, 7,9, 10,12, 3,6, 9,12, 3,7, 6,10, 9,21, 12,18, 7,10 + }, }; const std::array POSE_MAX_PEAKS{ POSE_MAX_PEOPLE, // BODY_25 @@ -380,6 +423,7 @@ namespace op POSE_MAX_PEOPLE, // BODY_65 POSE_MAX_PEOPLE, // CAR_12 POSE_MAX_PEOPLE, // BODY_25D + POSE_MAX_PEOPLE, // BODY_23 }; const std::array POSE_CCN_DECREASE_FACTOR{ 8.f, // BODY_25 @@ -395,30 +439,31 @@ namespace op 8.f, // BODY_65 8.f, // CAR_12 8.f, // BODY_25D + 8.f, // BODY_23 }; // Default Model Parameters // They might be modified on running time const auto nmsT = (COCO_CHALLENGE ? 0.02f : 0.05f); const std::array POSE_DEFAULT_NMS_THRESHOLD{ - nmsT, nmsT, 0.6f, 0.3f, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT + nmsT, nmsT, 0.6f, 0.3f, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT }; const auto minAT = (COCO_CHALLENGE ? 0.75f : 0.95f); // Matlab version: 0.85f const std::array POSE_DEFAULT_CONNECT_INTER_MIN_ABOVE_THRESHOLD{ - minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT + minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT // 0.85f, 0.85f, 0.85f, 0.85f, 0.85f, 0.85f // Matlab version }; const auto conIT = (COCO_CHALLENGE ? 0.01f : 0.05f); const std::array POSE_DEFAULT_CONNECT_INTER_THRESHOLD{ - conIT, conIT, 0.01f, 0.01f, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT + conIT, conIT, 0.01f, 0.01f, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT }; const auto minSC = (COCO_CHALLENGE ? 2 : 3); const std::array POSE_DEFAULT_CONNECT_MIN_SUBSET_CNT{ - minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC + minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC }; const auto minSS = (COCO_CHALLENGE ? 0.05f : 0.4f); const std::array POSE_DEFAULT_CONNECT_MIN_SUBSET_SCORE{ - minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS + minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS // 0.2f, 0.4f, 0.4f, 0.4f, 0.4f, 0.4f // Matlab version }; diff --git a/src/openpose/pose/poseParametersRender.cpp b/src/openpose/pose/poseParametersRender.cpp index c5241298..1e33537e 100644 --- a/src/openpose/pose/poseParametersRender.cpp +++ b/src/openpose/pose/poseParametersRender.cpp @@ -12,11 +12,12 @@ namespace op std::vector{POSE_BODY_19_SCALES_RENDER_GPU}, // BODY_19_X2 std::vector{POSE_BODY_59_SCALES_RENDER_GPU}, // BODY_59 std::vector{POSE_BODY_19_SCALES_RENDER_GPU}, // BODY_19N - std::vector{POSE_BODY_25E_SCALES_RENDER_GPU}, // BODY_25E + std::vector{POSE_BODY_25_SCALES_RENDER_GPU}, // BODY_25E std::vector{POSE_BODY_25_SCALES_RENDER_GPU}, // BODY_25_19 std::vector{POSE_BODY_65_SCALES_RENDER_GPU}, // BODY_65 std::vector{POSE_CAR_12_SCALES_RENDER_GPU}, // CAR_12 std::vector{POSE_BODY_25_SCALES_RENDER_GPU}, // BODY_25D + std::vector{POSE_BODY_23_SCALES_RENDER_GPU}, // BODY_23 }; const std::array, (int)PoseModel::Size> POSE_COLORS{ std::vector{POSE_BODY_25_COLORS_RENDER_GPU}, // BODY_25 @@ -27,11 +28,12 @@ namespace op std::vector{POSE_BODY_19_COLORS_RENDER_GPU}, // BODY_19_X2 std::vector{POSE_BODY_59_COLORS_RENDER_GPU}, // BODY_59 std::vector{POSE_BODY_19_COLORS_RENDER_GPU}, // BODY_19N - std::vector{POSE_BODY_25E_COLORS_RENDER_GPU}, // BODY_25E + std::vector{POSE_BODY_25_COLORS_RENDER_GPU}, // BODY_25E std::vector{POSE_BODY_25_COLORS_RENDER_GPU}, // BODY_25_19 std::vector{POSE_BODY_65_COLORS_RENDER_GPU}, // BODY_65 std::vector{POSE_CAR_12_COLORS_RENDER_GPU}, // CAR_12 std::vector{POSE_BODY_25_COLORS_RENDER_GPU}, // BODY_25D + std::vector{POSE_BODY_23_COLORS_RENDER_GPU}, // BODY_23 }; const std::array, (int)PoseModel::Size> POSE_BODY_PART_PAIRS_RENDER{ std::vector{POSE_BODY_25_PAIRS_RENDER_GPU}, // BODY_25 @@ -42,11 +44,12 @@ namespace op std::vector{POSE_BODY_19_PAIRS_RENDER_GPU}, // BODY_19_X2 std::vector{POSE_BODY_59_PAIRS_RENDER_GPU}, // BODY_59 std::vector{POSE_BODY_19_PAIRS_RENDER_GPU}, // BODY_19N - std::vector{POSE_BODY_25E_PAIRS_RENDER_GPU}, // BODY_25E + std::vector{POSE_BODY_25_PAIRS_RENDER_GPU}, // BODY_25E std::vector{POSE_BODY_25_PAIRS_RENDER_GPU}, // BODY_25_19 std::vector{POSE_BODY_65_PAIRS_RENDER_GPU}, // BODY_65 std::vector{POSE_CAR_12_PAIRS_RENDER_GPU}, // CAR_12 std::vector{POSE_BODY_25_PAIRS_RENDER_GPU}, // BODY_25D + std::vector{POSE_BODY_23_PAIRS_RENDER_GPU}, // BODY_23 }; // Rendering functions diff --git a/src/openpose/pose/renderPose.cu b/src/openpose/pose/renderPose.cu index e1a4ebdf..29f7fdd9 100644 --- a/src/openpose/pose/renderPose.cu +++ b/src/openpose/pose/renderPose.cu @@ -12,7 +12,7 @@ namespace op __constant__ const unsigned int BODY_25_PAIRS_GPU[] = {POSE_BODY_25_PAIRS_RENDER_GPU}; __constant__ const unsigned int COCO_PAIRS_GPU[] = {POSE_COCO_PAIRS_RENDER_GPU}; __constant__ const unsigned int BODY_19_PAIRS_GPU[] = {POSE_BODY_19_PAIRS_RENDER_GPU}; - __constant__ const unsigned int BODY_25E_PAIRS_GPU[] = {POSE_BODY_25E_PAIRS_RENDER_GPU}; + __constant__ const unsigned int BODY_23_PAIRS_GPU[] = {POSE_BODY_23_PAIRS_RENDER_GPU}; __constant__ const unsigned int BODY_59_PAIRS_GPU[] = {POSE_BODY_59_PAIRS_RENDER_GPU}; __constant__ const unsigned int BODY_65_PAIRS_GPU[] = {POSE_BODY_65_PAIRS_RENDER_GPU}; __constant__ const unsigned int MPI_PAIRS_GPU[] = {POSE_MPI_PAIRS_RENDER_GPU}; @@ -21,7 +21,7 @@ namespace op __constant__ const float BODY_25_SCALES[] = {POSE_BODY_25_SCALES_RENDER_GPU}; __constant__ const float COCO_SCALES[] = {POSE_COCO_SCALES_RENDER_GPU}; __constant__ const float BODY_19_SCALES[] = {POSE_BODY_19_SCALES_RENDER_GPU}; - __constant__ const float BODY_25E_SCALES[] = {POSE_BODY_25E_SCALES_RENDER_GPU}; + __constant__ const float BODY_23_SCALES[] = {POSE_BODY_23_SCALES_RENDER_GPU}; __constant__ const float BODY_59_SCALES[] = {POSE_BODY_59_SCALES_RENDER_GPU}; __constant__ const float BODY_65_SCALES[] = {POSE_BODY_65_SCALES_RENDER_GPU}; __constant__ const float MPI_SCALES[] = {POSE_MPI_SCALES_RENDER_GPU}; @@ -30,7 +30,7 @@ namespace op __constant__ const float BODY_25_COLORS[] = {POSE_BODY_25_COLORS_RENDER_GPU}; __constant__ const float COCO_COLORS[] = {POSE_COCO_COLORS_RENDER_GPU}; __constant__ const float BODY_19_COLORS[] = {POSE_BODY_19_COLORS_RENDER_GPU}; - __constant__ const float BODY_25E_COLORS[] = {POSE_BODY_25E_COLORS_RENDER_GPU}; + __constant__ const float BODY_23_COLORS[] = {POSE_BODY_23_COLORS_RENDER_GPU}; __constant__ const float BODY_59_COLORS[] = {POSE_BODY_59_COLORS_RENDER_GPU}; __constant__ const float BODY_65_COLORS[] = {POSE_BODY_65_COLORS_RENDER_GPU}; __constant__ const float MPI_COLORS[] = {POSE_MPI_COLORS_RENDER_GPU}; @@ -169,9 +169,9 @@ namespace op blendOriginalFrame, (googlyEyes ? 15 : -1), (googlyEyes ? 16 : -1)); } - __global__ void renderPoseBody25E(float* targetPtr, const int targetWidth, const int targetHeight, - const float* const posePtr, const int numberPeople, const float threshold, - const bool googlyEyes, const bool blendOriginalFrame, const float alphaColorToAdd) + __global__ void renderPoseBody23(float* targetPtr, const int targetWidth, const int targetHeight, + const float* const posePtr, const int numberPeople, const float threshold, + const bool googlyEyes, const bool blendOriginalFrame, const float alphaColorToAdd) { const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; @@ -183,17 +183,17 @@ namespace op __shared__ float sharedScaleF[POSE_MAX_PEOPLE]; // Other parameters - const auto numberPartPairs = sizeof(BODY_25E_PAIRS_GPU) / (2*sizeof(BODY_25E_PAIRS_GPU[0])); - const auto numberScales = sizeof(BODY_25E_SCALES) / sizeof(BODY_25E_SCALES[0]); - const auto numberColors = sizeof(BODY_25E_COLORS) / (3*sizeof(BODY_25E_COLORS[0])); + const auto numberPartPairs = sizeof(BODY_23_PAIRS_GPU) / (2*sizeof(BODY_23_PAIRS_GPU[0])); + const auto numberScales = sizeof(BODY_23_SCALES) / sizeof(BODY_23_SCALES[0]); + const auto numberColors = sizeof(BODY_23_COLORS) / (3*sizeof(BODY_23_COLORS[0])); const auto radius = fastMin(targetWidth, targetHeight) / 100.f; const auto lineWidth = fastMin(targetWidth, targetHeight) / 120.f; // Render key points renderKeypoints(targetPtr, sharedMaxs, sharedMins, sharedScaleF, globalIdx, x, y, targetWidth, targetHeight, - posePtr, BODY_25E_PAIRS_GPU, numberPeople, 25, numberPartPairs, BODY_25E_COLORS, numberColors, - radius, lineWidth, BODY_25E_SCALES, numberScales, threshold, alphaColorToAdd, - blendOriginalFrame, (googlyEyes ? 15 : -1), (googlyEyes ? 16 : -1)); + posePtr, BODY_23_PAIRS_GPU, numberPeople, 23, numberPartPairs, BODY_23_COLORS, numberColors, + radius, lineWidth, BODY_23_SCALES, numberScales, threshold, alphaColorToAdd, + blendOriginalFrame, (googlyEyes ? 13 : -1), (googlyEyes ? 14 : -1)); } __global__ void renderPoseBody25(float* targetPtr, const int targetWidth, const int targetHeight, @@ -543,7 +543,7 @@ namespace op // Body pose if (poseModel == PoseModel::BODY_25 || poseModel == PoseModel::BODY_25_19 - || poseModel == PoseModel::BODY_25D) + || poseModel == PoseModel::BODY_25D || poseModel == PoseModel::BODY_25E) renderPoseBody25<<>>( framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending @@ -559,8 +559,8 @@ namespace op framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending ); - else if (poseModel == PoseModel::BODY_25E) - renderPoseBody25E<<>>( + else if (poseModel == PoseModel::BODY_23) + renderPoseBody23<<>>( framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending ); diff --git a/src/openpose/producer/imageDirectoryReader.cpp b/src/openpose/producer/imageDirectoryReader.cpp index 028f146a..60239faa 100644 --- a/src/openpose/producer/imageDirectoryReader.cpp +++ b/src/openpose/producer/imageDirectoryReader.cpp @@ -127,7 +127,12 @@ namespace op { try { + // Read frame auto frame = loadImage(mFilePaths.at(mFrameNameCounter++).c_str(), CV_LOAD_IMAGE_COLOR); + // Skip frames if frame step > 1 + const auto frameStep = Producer::get(ProducerProperty::FrameStep); + if (frameStep > 1) + set(CV_CAP_PROP_POS_FRAMES, mFrameNameCounter + frameStep-1); // Check frame integrity. This function also checks width/height changes. However, if it is performed // after setWidth/setHeight this is performed over the new resolution (so they always match). checkFrameIntegrity(frame); diff --git a/src/openpose/producer/producer.cpp b/src/openpose/producer/producer.cpp index 4d761e58..bd221593 100644 --- a/src/openpose/producer/producer.cpp +++ b/src/openpose/producer/producer.cpp @@ -155,6 +155,15 @@ namespace op "ProducerProperty::Rotation only implemented for {0, 90, 180, 270} degrees.", __LINE__, __FUNCTION__, __FILE__); } + else if (property == ProducerProperty::FrameStep) + { + // Sanity check + if (value < 1) + { + const auto message = "The frame step must be greater than 0 (`--frame_step`). Use 1 by default."; + error(message, __LINE__, __FUNCTION__, __FILE__); + } + } // Common operation mProperties[(unsigned char)property] = value; diff --git a/src/openpose/producer/videoCaptureReader.cpp b/src/openpose/producer/videoCaptureReader.cpp index 626eb6ee..5aa0812e 100644 --- a/src/openpose/producer/videoCaptureReader.cpp +++ b/src/openpose/producer/videoCaptureReader.cpp @@ -82,8 +82,25 @@ namespace op { try { + // Get frame cv::Mat frame; mVideoCapture >> frame; + // Skip frames if frame step > 1 + const auto frameStep = Producer::get(ProducerProperty::FrameStep); + if (frameStep > 1 && !frame.empty() && get(CV_CAP_PROP_POS_FRAMES) < get(CV_CAP_PROP_FRAME_COUNT)-1) + { + // Close if end of video + if (get(CV_CAP_PROP_POS_FRAMES) + frameStep-1 >= get(CV_CAP_PROP_FRAME_COUNT)) + mVideoCapture.release(); + // Frame step usually more efficient if just reading sequentially + else if (frameStep < 51) + for (auto i = 1 ; i < frameStep ; i++) + mVideoCapture >> frame; + // Using set(CV_CAP_PROP_POS_FRAMES, value) is efficient only if step is big + else + set(CV_CAP_PROP_POS_FRAMES, get(CV_CAP_PROP_POS_FRAMES) + frameStep-1); + } + // Return frame return frame; } catch (const std::exception& e) diff --git a/src/openpose/utilities/flagsToOpenPose.cpp b/src/openpose/utilities/flagsToOpenPose.cpp index 5214ee02..fd538833 100644 --- a/src/openpose/utilities/flagsToOpenPose.cpp +++ b/src/openpose/utilities/flagsToOpenPose.cpp @@ -25,6 +25,8 @@ namespace op return PoseModel::MPI_15_4; else if (poseModeString == "BODY_19") return PoseModel::BODY_19; + else if (poseModeString == "BODY_23") + return PoseModel::BODY_23; else if (poseModeString == "BODY_25E") return PoseModel::BODY_25E; else if (poseModeString == "BODY_19N") diff --git a/src/openpose/wrapper/wrapperStructInput.cpp b/src/openpose/wrapper/wrapperStructInput.cpp index f740e095..bd382758 100644 --- a/src/openpose/wrapper/wrapperStructInput.cpp +++ b/src/openpose/wrapper/wrapperStructInput.cpp @@ -3,11 +3,12 @@ namespace op { WrapperStructInput::WrapperStructInput(const std::shared_ptr producerSharedPtr_, - const unsigned long long frameFirst_, const unsigned long long frameLast_, - const bool realTimeProcessing_, const bool frameFlip_, - const int frameRotate_, const bool framesRepeat_) : + const unsigned long long frameFirst_, const unsigned long long frameStep_, + const unsigned long long frameLast_, const bool realTimeProcessing_, + const bool frameFlip_, const int frameRotate_, const bool framesRepeat_) : producerSharedPtr{producerSharedPtr_}, frameFirst{frameFirst_}, + frameStep{frameStep_}, frameLast{frameLast_}, realTimeProcessing{realTimeProcessing_}, frameFlip{frameFlip_},