diff --git a/doc/demo_overview.md b/doc/demo_overview.md index 3472d990..78dc85f4 100644 --- a/doc/demo_overview.md +++ b/doc/demo_overview.md @@ -161,6 +161,7 @@ Each flag is divided into flag name, default value, and description. - DEFINE_int32(keypoint_scale, 0, "Scaling of the (x,y) coordinates of the final pose data array, i.e., the scale of the (x,y) coordinates that will be saved with the `write_json` & `write_keypoint` flags. Select `0` to scale it to the original source resolution; `1`to scale it to the net output size (set with `net_resolution`); `2` to scale it to the final output size (set with `resolution`); `3` to scale it in the range [0,1], where (0,0) would be the top-left corner of the image, and (1,1) the bottom-right one; and 4 for range [-1,1], where (-1,-1) would be the top-left corner of the image, and (1,1) the bottom-right one. Non related with `scale_number` and `scale_gap`."); - DEFINE_int32(number_people_max, -1, "This parameter will limit the maximum number of people detected, by keeping the people with top scores. The score is based in person area over the image, body part score, as well as joint score (between each pair of connected body parts). Useful if you know the exact number of people in the scene, so it can remove false positives (if all the people have been detected. However, it might also include false negatives by removing very small or highly occluded people. -1 will keep them all."); - DEFINE_bool(maximize_positives, false, "It reduces the thresholds to accept a person candidate. It highly increases both false and true positives. I.e., it maximizes average recall but could harm average precision."); +- DEFINE_double(fps_max, -1., "Maximum processing frame rate. By default (-1), OpenPose will process frames as fast as possible. Example usage: If OpenPose is displaying images too quickly, this can reduce the speed so the user can analyze better each frame from the GUI."); 4. OpenPose Body Pose - DEFINE_bool(body_disable, false, "Disable body keypoint detection. Option only possible for faster (but less accurate) face keypoint detection."); diff --git a/doc/release_notes.md b/doc/release_notes.md index 23e9baff..6eed2f98 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -291,6 +291,7 @@ OpenPose Library - Release Notes 21. Added find_package(Protobuf) to allow specific versions of Protobuf. 22. Examples do not end in core dumped if an OpenPose exception occurred during initialization, but it is rather closed returning -1. However, it will still results in core dumped if the exception occurs during multi-threading execution. 23. Video (`--write_video`) can be generated from images (`--image_dir`), as long as they maintain the same resolution. + 24. Added `--fps_max` flag to limit the maximum processing frame rate of OpenPose (useful to display results at a maximum desired speed). 2. Functions or parameters renamed: 1. By default, python example `tutorial_developer/python_2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368. 2. WrapperStructPose default parameters changed to match those of the OpenPose demo binary. diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index 91a66a4c..1a9c3af5 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -72,7 +72,8 @@ int openPoseDemo() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapper.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/examples/tutorial_add_module/1_custom_post_processing.cpp b/examples/tutorial_add_module/1_custom_post_processing.cpp index 4c227ffd..d11f3b9e 100644 --- a/examples/tutorial_add_module/1_custom_post_processing.cpp +++ b/examples/tutorial_add_module/1_custom_post_processing.cpp @@ -85,7 +85,8 @@ int tutorialAddModule1() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapperT.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp b/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp index 7299a967..24f36a1e 100644 --- a/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp +++ b/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp @@ -91,7 +91,8 @@ int tutorialApiCpp3() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapper.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp b/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp index fce642d5..e7692b92 100644 --- a/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp +++ b/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp @@ -224,7 +224,8 @@ int tutorialApiCpp4() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapperT.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp b/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp index 3b12621d..ea1889e1 100644 --- a/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp +++ b/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp @@ -167,7 +167,8 @@ int tutorialApiCpp5() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapperT.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp b/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp index d799cd6f..805b7cdb 100644 --- a/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp +++ b/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp @@ -135,7 +135,8 @@ int tutorialApiCpp6() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapperT.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp b/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp index 38faba10..024e4bd6 100644 --- a/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp +++ b/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp @@ -170,7 +170,8 @@ int tutorialApiCpp7() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapperT.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp b/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp index 7d57d55a..a78e4c3d 100644 --- a/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp +++ b/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp @@ -182,7 +182,8 @@ int tutorialApiCpp8() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapperT.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp b/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp index 2ee381e2..6a3ebcbb 100644 --- a/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp +++ b/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp @@ -281,7 +281,8 @@ int tutorialApiCpp9() FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, - (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, enableGoogleLogging}; + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + enableGoogleLogging}; opWrapperT.configure(wrapperStructPose); // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{ diff --git a/include/openpose/flags.hpp b/include/openpose/flags.hpp index ec9aef0e..7323c2e6 100644 --- a/include/openpose/flags.hpp +++ b/include/openpose/flags.hpp @@ -80,6 +80,9 @@ DEFINE_int32(number_people_max, -1, "This parameter will lim " highly occluded people. -1 will keep them all."); DEFINE_bool(maximize_positives, false, "It reduces the thresholds to accept a person candidate. It highly increases both false and" " true positives. I.e., it maximizes average recall but could harm average precision."); +DEFINE_double(fps_max, -1., "Maximum processing frame rate. By default (-1), OpenPose will process frames as fast as" + " possible. Example usage: If OpenPose is displaying images too quickly, this can reduce" + " the speed so the user can analyze better each frame from the GUI."); // OpenPose Body Pose DEFINE_bool(body_disable, false, "Disable body keypoint detection. Option only possible for faster (but less accurate) face" " keypoint detection."); diff --git a/include/openpose/thread/headers.hpp b/include/openpose/thread/headers.hpp index 15bab8b8..1afcad9e 100644 --- a/include/openpose/thread/headers.hpp +++ b/include/openpose/thread/headers.hpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/include/openpose/thread/wFpsMax.hpp b/include/openpose/thread/wFpsMax.hpp new file mode 100644 index 00000000..a00dd6de --- /dev/null +++ b/include/openpose/thread/wFpsMax.hpp @@ -0,0 +1,83 @@ +#ifndef OPENPOSE_THREAD_W_FPS_MAX_HPP +#define OPENPOSE_THREAD_W_FPS_MAX_HPP + +#include +#include +#include +#include + +namespace op +{ + template + class WFpsMax : public Worker + { + public: + explicit WFpsMax(const double fpsMax); + + virtual ~WFpsMax(); + + void initializationOnThread(); + + void work(TDatums& tDatums); + + private: + const unsigned long long mNanosecondsToSleep; + + DELETE_COPY(WFpsMax); + }; +} + + + + + +// Implementation +namespace op +{ + template + WFpsMax::WFpsMax(const double fpsMax) : + mNanosecondsToSleep{uLongLongRound(1e9/fpsMax)} + { + } + + template + WFpsMax::~WFpsMax() + { + } + + template + void WFpsMax::initializationOnThread() + { + } + + template + void WFpsMax::work(TDatums& tDatums) + { + try + { + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + // tDatums not used --> Avoid warning + UNUSED(tDatums); + // Sleep the desired time + std::this_thread::sleep_for(std::chrono::nanoseconds{mNanosecondsToSleep}); + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + this->stop(); + tDatums = nullptr; + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + COMPILE_TEMPLATE_DATUM(WFpsMax); +} + +#endif // OPENPOSE_THREAD_W_FPS_MAX_HPP diff --git a/include/openpose/wrapper/wrapperAuxiliary.hpp b/include/openpose/wrapper/wrapperAuxiliary.hpp index cfcb4dfa..a0384eeb 100644 --- a/include/openpose/wrapper/wrapperAuxiliary.hpp +++ b/include/openpose/wrapper/wrapperAuxiliary.hpp @@ -105,18 +105,6 @@ namespace op auto wrapperStructPose = wrapperStructPoseTemp; auto multiThreadEnabled = multiThreadEnabledTemp; - // Workers - TWorker datumProducerW; - TWorker scaleAndSizeExtractorW; - TWorker cvMatToOpInputW; - TWorker cvMatToOpOutputW; - std::vector> poseExtractorsWs; - std::vector> poseTriangulationsWs; - std::vector> jointAngleEstimationsWs; - std::vector postProcessingWs; - std::vector outputWs; - TWorker guiW; - // User custom workers const auto& userInputWs = userWs[int(WorkerType::Input)]; const auto& userPostProcessingWs = userWs[int(WorkerType::PostProcessing)]; @@ -219,6 +207,7 @@ namespace op } // Producer + TWorker datumProducerW; if (oPProducer) { const auto datumProducer = std::make_shared>( @@ -235,6 +224,14 @@ namespace op std::vector> handExtractorNets; std::vector> poseGpuRenderers; std::shared_ptr poseCpuRenderer; + // Workers + TWorker scaleAndSizeExtractorW; + TWorker cvMatToOpInputW; + TWorker cvMatToOpOutputW; + std::vector> poseExtractorsWs; + std::vector> poseTriangulationsWs; + std::vector> jointAngleEstimationsWs; + std::vector postProcessingWs; if (numberThreads > 0) { // Get input scales and sizes @@ -582,7 +579,7 @@ namespace op #endif // Output workers - outputWs.clear(); + std::vector outputWs; // Print verbose if (wrapperStructOutput.verbose > 0.) { @@ -700,7 +697,7 @@ namespace op outputWs.emplace_back(std::make_shared>(guiInfoAdder)); } // Minimal graphical user interface (GUI) - guiW = nullptr; + TWorker guiW; if (guiEnabled) { // PoseRenderers to Renderers @@ -753,6 +750,10 @@ namespace op else error("Unknown DisplayMode.", __LINE__, __FUNCTION__, __FILE__); } + // Set FpsMax + TWorker wFpsMax; + if (wrapperStructPose.fpsMax > 0.) + wFpsMax = std::make_shared>(wrapperStructPose.fpsMax); // Set wrapper as configured log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); @@ -986,6 +987,13 @@ namespace op threadIdPP(threadId, multiThreadEnabled); } log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + // Setting maximum speed + if (wFpsMax != nullptr) + { + log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + threadManager.add(threadId, wFpsMax, queueIn++, queueOut++); + threadIdPP(threadId, multiThreadEnabled); + } } catch (const std::exception& e) { diff --git a/include/openpose/wrapper/wrapperStructPose.hpp b/include/openpose/wrapper/wrapperStructPose.hpp index 1327038f..41e462ca 100644 --- a/include/openpose/wrapper/wrapperStructPose.hpp +++ b/include/openpose/wrapper/wrapperStructPose.hpp @@ -165,6 +165,14 @@ namespace op */ bool maximizePositives; + /** + * Maximum processing frame rate. + * By default (-1), OpenPose will process frames as fast as possible. + * Example usage: If OpenPose is displaying images too quickly, this can reduce the speed so the user can + * analyze better each frame from the GUI. + */ + double fpsMax; + /** * Whether to internally enable Google Logging. * This option is only applicable if Caffe is used. @@ -190,7 +198,7 @@ namespace op const std::string& modelFolder = "models/", const std::vector& heatMapTypes = {}, const ScaleMode heatMapScale = ScaleMode::ZeroToOne, const bool addPartCandidates = false, const float renderThreshold = 0.05f, const int numberPeopleMax = -1, const bool maximizePositives = false, - const bool enableGoogleLogging = true); + const double fpsMax = -1., const bool enableGoogleLogging = true); }; } diff --git a/src/openpose/thread/defineTemplates.cpp b/src/openpose/thread/defineTemplates.cpp index 6e38caa6..9ff32c1e 100644 --- a/src/openpose/thread/defineTemplates.cpp +++ b/src/openpose/thread/defineTemplates.cpp @@ -2,20 +2,26 @@ namespace op { + // Queues DEFINE_TEMPLATE_DATUM(PriorityQueue); DEFINE_TEMPLATE_DATUM(Queue); template class OP_API QueueBase>; template class OP_API QueueBase, std::greater>>; + // Subthread DEFINE_TEMPLATE_DATUM(SubThread); DEFINE_TEMPLATE_DATUM(SubThreadNoQueue); DEFINE_TEMPLATE_DATUM(SubThreadQueueIn); DEFINE_TEMPLATE_DATUM(SubThreadQueueInOut); DEFINE_TEMPLATE_DATUM(SubThreadQueueOut); + // Thread DEFINE_TEMPLATE_DATUM(Thread); DEFINE_TEMPLATE_DATUM(ThreadManager); + // Main workers DEFINE_TEMPLATE_DATUM(Worker); DEFINE_TEMPLATE_DATUM(WorkerConsumer); DEFINE_TEMPLATE_DATUM(WorkerProducer); + // W-classes + DEFINE_TEMPLATE_DATUM(WFpsMax); DEFINE_TEMPLATE_DATUM(WIdGenerator); template class OP_API WQueueAssembler; DEFINE_TEMPLATE_DATUM(WQueueOrderer); diff --git a/src/openpose/wrapper/wrapperStructPose.cpp b/src/openpose/wrapper/wrapperStructPose.cpp index b2d10f5c..317077d8 100644 --- a/src/openpose/wrapper/wrapperStructPose.cpp +++ b/src/openpose/wrapper/wrapperStructPose.cpp @@ -9,7 +9,8 @@ namespace op const bool blendOriginalFrame_, const float alphaKeypoint_, const float alphaHeatMap_, const int defaultPartToRender_, const std::string& modelFolder_, const std::vector& heatMapTypes_, const ScaleMode heatMapScale_, const bool addPartCandidates_, const float renderThreshold_, - const int numberPeopleMax_, const bool maximizePositives_, const bool enableGoogleLogging_) : + const int numberPeopleMax_, const bool maximizePositives_, const double fpsMax_, + const bool enableGoogleLogging_) : enable{enable_}, netInputSize{netInputSize_}, outputSize{outputSize_}, @@ -31,6 +32,7 @@ namespace op renderThreshold{renderThreshold_}, numberPeopleMax{numberPeopleMax_}, maximizePositives{maximizePositives_}, + fpsMax{fpsMax_}, enableGoogleLogging{enableGoogleLogging_} { }