From 75ea8eb538cd41bc21d368fdae97c42cc37cce8a Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Sat, 8 Jul 2017 22:30:02 -0400 Subject: [PATCH] Version 1.0.0: Hands added --- Makefile | 2 +- README.md | 17 +- doc/demo_overview.md | 63 +- doc/installation.md | 2 +- doc/output.md | 3 +- doc/release_notes.md | 28 +- examples/openpose/openpose.cpp | 58 +- .../tutorial_pose/1_extract_from_image.cpp | 23 +- .../2_extract_pose_or_heatmat_from_image.cpp | 24 +- .../tutorial_wrapper/1_user_asynchronous.cpp | 59 +- .../tutorial_wrapper/2_user_synchronous.cpp | 59 +- include/openpose/core/datum.hpp | 2 +- include/openpose/core/enumClasses.hpp | 8 - include/openpose/face/faceParameters.hpp | 1 - include/openpose/face/faceRenderer.hpp | 7 +- include/openpose/face/renderFace.hpp | 6 +- include/openpose/hand/handExtractor.hpp | 5 +- include/openpose/hand/handParameters.hpp | 5 +- include/openpose/hand/handRenderer.hpp | 7 +- include/openpose/hand/renderHand.hpp | 8 +- include/openpose/pose/poseParameters.hpp | 1 - include/openpose/pose/poseRenderer.hpp | 10 +- include/openpose/pose/renderPose.hpp | 27 +- .../openpose/utilities/flagsToOpenPose.hpp | 2 - include/openpose/wrapper/wrapper.hpp | 27 +- .../openpose/wrapper/wrapperStructFace.hpp | 10 +- .../openpose/wrapper/wrapperStructHand.hpp | 34 +- .../openpose/wrapper/wrapperStructPose.hpp | 10 +- models/getModels.bat | 8 + models/getModels.sh | 6 + models/hand/pose_deploy.prototxt | 1756 +++++++++++++++++ src/openpose/core/cvMatToOpInput.cpp | 2 +- src/openpose/core/resizeAndMergeBase.cu | 2 +- src/openpose/face/faceRenderer.cpp | 12 +- src/openpose/face/renderFace.cpp | 4 +- src/openpose/face/renderFace.cu | 18 +- src/openpose/hand/handExtractor.cpp | 219 +- src/openpose/hand/handRenderer.cpp | 10 +- src/openpose/hand/renderHand.cpp | 9 +- src/openpose/hand/renderHand.cu | 6 +- src/openpose/pose/poseRenderer.cpp | 11 +- src/openpose/pose/renderPose.cpp | 6 +- src/openpose/pose/renderPose.cu | 11 +- src/openpose/utilities/flagsToOpenPose.cpp | 35 - src/openpose/wrapper/wrapperStructFace.cpp | 6 +- src/openpose/wrapper/wrapperStructHand.cpp | 13 +- src/openpose/wrapper/wrapperStructPose.cpp | 5 +- 47 files changed, 2332 insertions(+), 315 deletions(-) create mode 100755 models/hand/pose_deploy.prototxt diff --git a/Makefile b/Makefile index e94c768e..e4b40356 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ LIB_BUILD_DIR := $(BUILD_DIR)/lib STATIC_NAME := $(LIB_BUILD_DIR)/lib$(LIBRARY_NAME).a DYNAMIC_VERSION_MAJOR := 1 DYNAMIC_VERSION_MINOR := 0 -DYNAMIC_VERSION_REVISION := 0-rc3 +DYNAMIC_VERSION_REVISION := 0 DYNAMIC_NAME_SHORT := lib$(LIBRARY_NAME).so #DYNAMIC_SONAME_SHORT := $(DYNAMIC_NAME_SHORT).$(DYNAMIC_VERSION_MAJOR) DYNAMIC_VERSIONED_NAME_SHORT := $(DYNAMIC_NAME_SHORT).$(DYNAMIC_VERSION_MAJOR).$(DYNAMIC_VERSION_MINOR).$(DYNAMIC_VERSION_REVISION) diff --git a/README.md b/README.md index ad010928..588e8cf4 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ OpenPose - May 2017: Windows version released! - Jun 2017: Face released! - Jul 2017: Easier Windows installation! +- Jul 2017: Hands released! - Check all the [release notes](doc/release_notes.md). - Interested in an internship on CMU as OpenPose programmer? See [this link](https://docs.google.com/document/d/14SygG39NjIRZfx08clewTdFMGwVdtRu2acyCi3TYcHs/edit?usp=sharing) for details. @@ -14,7 +15,7 @@ OpenPose ## Operating Systems 1. **Ubuntu** 14 and 16. 2. **Windows** 10. -3. Other people have been able to install it on **Windows 7 and 8**, **Mac**, **CentOS**, and **Nvidia Jetson (TK1 and TX1)** embedded systems. However, we do not support them at the moment. +3. Other people have been able to install it on **Windows 7 and 8**, **Mac**, **CentOS**, and **Nvidia Jetson (TK1 and TX1)** embedded systems. However, we do not officially support them at the moment. @@ -25,7 +26,7 @@ OpenPose is a **library for real-time multi-person keypoint detection and multi- OpenPose represents the **first real-time system to jointly detect human body, hand and facial keypoints (in total 130 keypoints) on single images**. In addition, the system computational performance on body keypoint estimation is invariant to the number of detected people in the image. -OpenPose is freely available for free non-commercial use, and may be redistributed under these conditions. Please, see the [license](LICENSE) for further details. Contact [Yaser Sheikh](http://www.cs.cmu.edu/~yaser/) for commercial purposes. +OpenPose is freely available for free non-commercial use, and may be redistributed under these conditions. Please, see the [license](LICENSE) for further details. For commercial purposes, contact [Yaser Sheikh](http://www.cs.cmu.edu/~yaser/). In addition, OpenPose would not be possible without the [CMU Panoptic Studio](http://domedb.perception.cs.cmu.edu/). @@ -54,6 +55,11 @@ The pose estimation work is based on the C++ code from [the ECCV 2016 demo](http ## Results +### Body + Hands + Face Estimation +

+ +

+ ### Body Estimation

@@ -64,13 +70,6 @@ The pose estimation work is based on the C++ code from [the ECCV 2016 demo](http

-## Coming Soon (But Already Working!) - -### Body + Hands + Face Estimation -

- -

- ### Body + Hands

diff --git a/doc/demo_overview.md b/doc/demo_overview.md index 33071e84..fb461752 100644 --- a/doc/demo_overview.md +++ b/doc/demo_overview.md @@ -28,12 +28,12 @@ Each flag is divided into flag name, default value, and description. - DEFINE_string(resolution, "1280x720", "The image resolution (display and output). Use \"-1x-1\" to force the program to use the default images resolution."); - DEFINE_int32(num_gpu, -1, "The number of GPU devices to use. If negative, it will use all the available GPUs in your machine."); - DEFINE_int32(num_gpu_start, 0, "GPU device start number."); -- DEFINE_int32(keypoint_scale, 0, "Scaling of the (x,y) coordinates of the final pose data array, i.e. the scale of the (x,y) coordinates that will be saved with the `write_keypoint` & `write_keypoint_json` flags. Select `0` to scale it to the original source resolution, `1`to scale it to the net output size (set with `net_resolution`), `2` to scale it to the final output size (set with `resolution`), `3` to scale it in the range [0,1], and 4 for range [-1,1]. Non related with `num_scales` and `scale_gap`."); +- DEFINE_int32(keypoint_scale, 0, "Scaling of the (x,y) coordinates of the final pose data array, i.e. the scale of the (x,y) coordinates that will be saved with the `write_keypoint` & `write_keypoint_json` flags. Select `0` to scale it to the original source resolution, `1`to scale it to the net output size (set with `net_resolution`), `2` to scale it to the final output size (set with `resolution`), `3` to scale it in the range [0,1], and 4 for range [-1,1]. Non related with `scale_number` and `scale_gap`."); 4. OpenPose Body Pose - DEFINE_string(model_pose, "COCO", "Model to be used (e.g. COCO, MPI, MPI_4_layers)."); - DEFINE_string(net_resolution, "656x368", "Multiples of 16. If it is increased, the accuracy usually increases. If it is decreased, the speed increases."); -- DEFINE_int32(num_scales, 1, "Number of scales to average."); -- DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless num_scales>1. Initial scale is always 1. If you want to change the initial scale, you actually want to multiply the `net_resolution` by your desired initial scale."); +- DEFINE_int32(scale_number, 1, "Number of scales to average."); +- DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1. If you want to change the initial scale, you actually want to multiply the `net_resolution` by your desired initial scale."); - DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array (program speed will decrease). Not required for our library, enable it only if you intend to process this information later. If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential memory order: body parts + bkg + PAFs. It will follow the order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`."); - DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to background."); - DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs."); @@ -42,23 +42,28 @@ Each flag is divided into flag name, default value, and description. - DEFINE_string(face_net_resolution, "368x368", "Multiples of 16. Analogous to `net_resolution` but applied to the face keypoint detector. 320x320 usually works fine while giving a substantial speed up when multiple faces on the image."); 6. OpenPose Hand - DEFINE_bool(hand, false, "Enables hand keypoint detection. It will share some parameters from the body pose, e.g. `model_folder`."); -- DEFINE_string(hand_net_resolution, "368x368", "Multiples of 16. Analogous to `net_resolution` but applied to the hand keypoint detector. 320x320 usually works fine while giving a substantial speed up when multiple hands on the image.");t_resolution` but applied to the hand keypoint detector."); -- DEFINE_int32(hand_detection_mode, -1, "Set to 0 to perform 1-time keypoint detection (fastest), 1 for iterative detection (recommended for images and fast videos, slow method), 2 for tracking (recommended for webcam if the frame rate is >10 FPS per GPU used and for video, in practice as fast as 1-time detection), 3 for both iterative and tracking (recommended for webcam if the resulting frame rate is still >10 FPS and for video, ideally best result but slower), or -1 (default) for automatic selection (fast method for webcam, tracking for video and iterative for images)."); +- DEFINE_string(hand_net_resolution, "368x368", "Multiples of 16. Analogous to `net_resolution` but applied to the hand keypoint detector."); +- DEFINE_int32(hand_scale_number, 1, "Analogous to `scale_number` but applied to the hand keypoint detector. Our best results were found with `hand_scale_number` = 6 and `hand_scale_range` = 0.4"); +- DEFINE_double(hand_scale_range, 0.4, "Analogous purpose than `scale_gap` but applied to the hand keypoint detector. Total range between smallest and biggest scale. The scales will be centered in ratio 1. E.g. if scaleRange = 0.4 and scalesNumber = 2, then there will be 2 scales, 0.8 and 1.2."); +- DEFINE_bool(hand_tracking, false, "Adding hand tracking might improve hand keypoints detection for webcam (if the frame rate is high enough, i.e. >7 FPS per GPU) and video. This is not person ID tracking, it simply looks for hands in positions at which hands were located in previous frames, but it does not guarantee the same person ID among frames"); 7. OpenPose Rendering - DEFINE_int32(part_to_show, 0, "Part to show from the start."); - DEFINE_bool(disable_blending, false, "If blending is enabled, it will merge the results with the original frame. If disabled, it will only display the results."); 8. OpenPose Rendering Pose +- DEFINE_double(render_threshold, 0.05, "Only estimated keypoints whose score confidences are higher than this threshold will be rendered. Generally, a high threshold (> 0.5) will only render very clear body parts; while small thresholds (~0.1) will also output guessed and occluded keypoints, but also more false positives (i.e. wrong detections)."); - DEFINE_int32(render_pose, 2, "Set to 0 for no rendering, 1 for CPU rendering (slightly faster), and 2 for GPU rendering (slower but greater functionality, e.g. `alpha_X` flags). If rendering is enabled, it will render both `outputData` and `cvOutputData` with the original image and desired body part to be shown (i.e. keypoints, heat maps or PAFs)."); - DEFINE_double(alpha_pose, 0.6, "Blending factor (range 0-1) for the body part rendering. 1 will show it completely, 0 will hide it. Only valid for GPU rendering."); - DEFINE_double(alpha_heatmap, 0.7, "Blending factor (range 0-1) between heatmap and original frame. 1 will only show the heatmap, 0 will only show the frame. Only valid for GPU rendering."); 9. OpenPose Rendering Face -- DEFINE_int32(render_face, -1, "Analogous to `render_pose` but applied to the face. Extra option: -1 to use the same configuration that `render_pose` is using."); -- DEFINE_double(alpha_face, 0.6, "Analogous to `alpha_pose` but applied to face."); -- DEFINE_double(alpha_heatmap_face, 0.7, "Analogous to `alpha_heatmap` but applied to face."); +- DEFINE_double(face_render_threshold, 0.4, "Analogous to `render_threshold`, but applied to the face keypoints."); +- DEFINE_int32(face_render, -1, "Analogous to `render_pose` but applied to the face. Extra option: -1 to use the same configuration that `render_pose` is using."); +- DEFINE_double(face_alpha_pose, 0.6, "Analogous to `alpha_pose` but applied to face."); +- DEFINE_double(face_alpha_heatmap, 0.7, "Analogous to `alpha_heatmap` but applied to face."); 10. OpenPose Rendering Hand -- DEFINE_int32(render_hand, -1, "Analogous to `render_pose` but applied to the hand. Extra option: -1 to use the same configuration that `render_pose` is using."); -- DEFINE_double(alpha_hand, 0.6, "Analogous to `alpha_pose` but applied to hand."); -- DEFINE_double(alpha_heatmap_hand, 0.7, "Analogous to `alpha_heatmap` but applied to hand."); +- DEFINE_double(hand_render_threshold, 0.2, "Analogous to `render_threshold`, but applied to the hand keypoints."); +- DEFINE_int32(hand_render, -1, "Analogous to `render_pose` but applied to the hand. Extra option: -1 to use the same configuration that `render_pose` is using."); +- DEFINE_double(hand_alpha_pose, 0.6, "Analogous to `alpha_pose` but applied to hand."); +- DEFINE_double(hand_alpha_heatmap, 0.7, "Analogous to `alpha_heatmap` but applied to hand."); 11. Display - DEFINE_bool(fullscreen, false, "Run in full-screen mode (press f during runtime to toggle)."); - DEFINE_bool(process_real_time, false, "Enable to keep the original source frame rate (e.g. for video). If the processing time is too long, it will skip frames. If it is too fast, it will slow it down."); @@ -76,7 +81,7 @@ Each flag is divided into flag name, default value, and description. - DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`. Recommended `png` or any compressed and lossless format."); ## Multiple Scales -Running at multiple scales might drastically slow down the speed, but it will increase the accuracy. Given the CNN input size (set with `net_resolution`), `num_scales` and `scale_gap` configure the number of scales to use and the gap between them, respectively. For instance, `--num_scales 3 --scale_gap 0.15` means using 3 scales at resolution: (1), (1-0.15) and (1-2*0.15) times the `net_resolution`. +Running at multiple scales might drastically slow down the speed, but it will increase the accuracy. Given the CNN input size (set with `net_resolution`), `scale_number` and `scale_gap` configure the number of scales to use and the gap between them, respectively. For instance, `--scale_number 3 --scale_gap 0.15` means using 3 scales at resolution: (1), (1-0.15) and (1-2*0.15) times the `net_resolution`. ## Heat Maps Storing The following command will save all the body part heat maps, background heat map and Part Affinity Fields (PAFs) in the folder `output_heatmaps_folder`. It will save them on PNG format. Instead of individually saving each of the 67 heatmaps (18 body parts + background + 2 x 19 PAFs) individually, the library concatenate them vertically into a huge (width x #heatmaps) x (height) matrix. The PAFs channels are multiplied by 2 because there is one heatmpa for the x-coordinates and one for the y-coordinates. The order is body parts + bkg + PAFs. It will follow the sequence on POSE_BODY_PART_MAPPING in [include/openpose/pose/poseParameters.hpp](../include/openpose/pose/poseParameters.hpp). @@ -102,7 +107,7 @@ Please, in order to check all the real time pose demo options and their details, - `--part_to_show`: Select the prediction channel to visualize (default: 0). 0 to visualize all the body parts, 1-18 for each body part heat map, 19 for the background heat map, 20 for all the body part heat maps together, 21 for all the PAFs, 22-69 for each body part pair PAF. - `--no_display`: Display window not opened. Useful if there is no X server and/or to slightly speed up the processing if visual output is not required. - `--num_gpu 2 --num_gpu_start 1`: Parallelize over this number of GPUs starting by the desired device id. Default `num_gpu` is -1, which will use all the available GPUs. -- `--num_scales 3 --scale_gap 0.15`: Use 3 scales, 1, (1-0.15), (1-0.15*2). Default is one scale. If you want to change the initial scale, you actually want to multiply your desired initial scale by the `net_resolution`. +- `--scale_number 3 --scale_gap 0.15`: Use 3 scales, 1, (1-0.15), (1-0.15*2). Default is one scale. If you want to change the initial scale, you actually want to multiply your desired initial scale by the `net_resolution`. - `--net_resolution 656x368 --resolution 1280x720`: For HD images and video (default values). - `--net_resolution 496x368 --resolution 640x480`: For VGA images and video. - `--model_pose MPI`: It will use MPI (15 body keypoints). Default: COCO (18 body keypoints). MPI is slightly faster. The variation `MPI_4_layers` sacrifies accuracy in order to further increase speed. @@ -111,29 +116,15 @@ Please, in order to check all the real time pose demo options and their details, ## Hands -Very important note, use `hand_detection_mode` accordingly. ``` -# Images # Fast method for speed -./build/examples/openpose/openpose.bin --hand --hand_detection_mode 0 -# Iterative for higher accuracy -./build/examples/openpose/openpose.bin --hand --hand_detection_mode 1 - -# Video -# Iterative tracking for higher accuracy -./build/examples/openpose/openpose.bin --video examples/media/video.avi --hand --hand_detection_mode 3 -# Tracking for speed -./build/examples/openpose/openpose.bin --video examples/media/video.avi --hand --hand_detection_mode 2 - -# Webcam -# Fast method for speed if the frame rate is low -./build/examples/openpose/openpose.bin --hand --hand_detection_mode 0 -# Iterative for higher accuracy (but the frame rate will be reduced) -./build/examples/openpose/openpose.bin --hand --hand_detection_mode 1 -# Tracking for higher accuracy if the frame rate is high enough. Worse results than fast method if frame rate is low -./build/examples/openpose/openpose.bin --hand --hand_detection_mode 2 -# Iterative + tracking for best accuracy if frame rate is high enough. Worse results than fast method if frame rate is low -./build/examples/openpose/openpose.bin --hand --hand_detection_mode 3 +./build/examples/openpose/openpose.bin --hand +# Best results found with 6 scales +./build/examples/openpose/openpose.bin --hand --hand_scale_number 6 --hand_scale_range 0.4 +# Adding tracking to Webcam (if FPS per GPU > 10 FPS) and Video +./build/examples/openpose/openpose.bin --video examples/media/video.avi --hand --hand_tracking +# Multi-scale + tracking is also possible +./build/examples/openpose/openpose.bin --video examples/media/video.avi --hand --hand_scale_number 6 --hand_scale_range 0.4 --hand_tracking ``` @@ -158,9 +149,9 @@ Very important note, use `hand_detection_mode` accordingly. ## Rendering Face without Pose ``` # CPU rendering (faster) -./build/examples/openpose/openpose.bin --face --render_pose 0 --render_face 1 +./build/examples/openpose/openpose.bin --face --render_pose 0 --face_render 1 # GPU rendering -./build/examples/openpose/openpose.bin --face --render_pose 0 --render_face 2 +./build/examples/openpose/openpose.bin --face --render_pose 0 --face_render 2 ``` diff --git a/doc/installation.md b/doc/installation.md index 14073a52..065cebaa 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -17,7 +17,7 @@ OpenPose Library - Compilation and Installation - At least 2 GB of free RAM memory. - Highly recommended: A CPU with at least 8 cores. -Note: These requirements assume the default configuration (i.e. `--net_resolution "656x368"` and `num_scales 1`). You might need more (with a greater net resolution and/or number of scales) or less resources (with smaller net resolution and/or using the MPI and MPI_4 models). +Note: These requirements assume the default configuration (i.e. `--net_resolution "656x368"` and `scale_number 1`). You might need more (with a greater net resolution and/or number of scales) or less resources (with smaller net resolution and/or using the MPI and MPI_4 models). diff --git a/doc/output.md b/doc/output.md index fc1bd860..df6600d8 100644 --- a/doc/output.md +++ b/doc/output.md @@ -68,7 +68,8 @@ There are 3 different keypoint Array elements on this class: const auto y = poseKeypoints[{person, part, 1}]; const auto score = poseKeypoints[{person, part, 2}]; // Slightly more efficient version - // If you want to access these elements on a huge loop, it is slightly faster (but usually not faster enough to be worthy) to get the index by your own + // If you want to access these elements on a huge loop, you can get the index + // by your own, but it is usually not faster enough to be worthy const auto baseIndex = poseKeypoints.getSize(2)*(person*numberBodyParts + part); const auto x = poseKeypoints[baseIndex]; const auto y = poseKeypoints[baseIndex + 1]; diff --git a/doc/release_notes.md b/doc/release_notes.md index bfffb393..250a69cf 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -46,21 +46,25 @@ OpenPose Library - Release Notes ## OpenPose 1.0.0 1. Main improvements: - 1. Windows branch merged to master branch. - 2. Face and hands use `Maximum` instead of `Nms`, since there is only 1 person / detection. - 3. Increased accuracy on multi-scale (added `Datum::scaleRatios` to save the relative scale ratio when multi-scale). - 4. Increased speed ~5% by adding CPU rendering (but GPU is the default rendering). - 5. Rendering colors modified, visually better results. - 6. Check() functions give more feedback. - 7. WCocoJsonSaver finished and removed its 3599-image limit. - 8. Added `camera_fps` so generated video will use that frame rate. - 9. Reduced the number of printed information messages. Default logging priority threshold increased to Priority::Max. - 10. Google flags to OpenPose configuration parameters reader moved from each demo to utilities/flagsToOpenPose. - 11. Nms classes do not use `numberParts` for `Reshape`, they deduce the value. - 12. Improved documentation. + 1. Added hand keypoint detection. + 2. Windows branch merged to master branch. + 3. Face and hands use `Maximum` instead of `Nms`, since there is only 1 person / detection. + 4. Increased accuracy on multi-scale (added `Datum::scaleRatios` to save the relative scale ratio when multi-scale). + 5. Increased speed ~5% by adding CPU rendering (but GPU is the default rendering). + 6. Rendering colors modified, visually better results. + 7. Rendering threshold for pose, face and hands becomes user-configurable. + 8. Check() functions give more feedback. + 9. WCocoJsonSaver finished and removed its 3599-image limit. + 10. Added `camera_fps` so generated video will use that frame rate. + 11. Reduced the number of printed information messages. Default logging priority threshold increased to Priority::Max. + 12. Google flags to OpenPose configuration parameters reader moved from each demo to utilities/flagsToOpenPose. + 13. Nms classes do not use `numberParts` for `Reshape`, they deduce the value. + 14. Improved documentation. 2. Functions or parameters renamed: 1. Render flags renamed in the demo in order to incorporate the CPU/GPU rendering. 2. Keypoints saved in JSON files (`write_keypoint_json`) are now saved as `pose_keypoints`, `face_keypoints`, `hand_left_keypoints`, and `hand_right_keypoints`. They all were previously saved as `body_parts`. + 3. Flag `num_scales` renamed as `scale_number`. + 4. All hand and pose flags renamed such as they start by `hand_` and `face_` respectively. 3. Main bugs fixed: 1. Fixed bug in Array::getConstCvMat() if mVolume=0, now returning empty cv::Mat. 2. Fixed bug: `--process_real_time` threw error with webcam. diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index 14a87e9c..f34b67ba 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -73,15 +73,15 @@ DEFINE_int32(keypoint_scale, 0, "Scaling of the (x,y) co " Select `0` to scale it to the original source resolution, `1`to scale it to the net output" " size (set with `net_resolution`), `2` to scale it to the final output size (set with" " `resolution`), `3` to scale it in the range [0,1], and 4 for range [-1,1]. Non related" - " with `num_scales` and `scale_gap`."); + " with `scale_number` and `scale_gap`."); // OpenPose Body Pose DEFINE_string(model_pose, "COCO", "Model to be used (e.g. COCO, MPI, MPI_4_layers)."); DEFINE_string(net_resolution, "656x368", "Multiples of 16. If it is increased, the accuracy usually increases. If it is decreased," " the speed increases."); -DEFINE_int32(num_scales, 1, "Number of scales to average."); -DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless num_scales>1. Initial scale is always 1. If you" - " want to change the initial scale, you actually want to multiply the `net_resolution` by" - " your desired initial scale."); +DEFINE_int32(scale_number, 1, "Number of scales to average."); +DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." + " If you want to change the initial scale, you actually want to multiply the" + " `net_resolution` by your desired initial scale."); DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array" " (program speed will decrease). Not required for our library, enable it only if you intend" " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" @@ -100,18 +100,24 @@ DEFINE_string(face_net_resolution, "368x368", "Multiples of 16. Analog DEFINE_bool(hand, false, "Enables hand keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`."); DEFINE_string(hand_net_resolution, "368x368", "Multiples of 16. Analogous to `net_resolution` but applied to the hand keypoint detector."); -DEFINE_int32(hand_detection_mode, -1, "Set to 0 to perform 1-time keypoint detection (fastest), 1 for iterative detection" - " (recommended for images and fast videos, slow method), 2 for tracking (recommended for" - " webcam if the frame rate is >10 FPS per GPU used and for video, in practice as fast as" - " 1-time detection), 3 for both iterative and tracking (recommended for webcam if the" - " resulting frame rate is still >10 FPS and for video, ideally best result but slower), or" - " -1 (default) for automatic selection (fast method for webcam, tracking for video and" - " iterative for images)."); +DEFINE_int32(hand_scale_number, 1, "Analogous to `scale_number` but applied to the hand keypoint detector. Our best results" + " were found with `hand_scale_number` = 6 and `hand_scale_range` = 0.4"); +DEFINE_double(hand_scale_range, 0.4, "Analogous purpose than `scale_gap` but applied to the hand keypoint detector. Total range" + " between smallest and biggest scale. The scales will be centered in ratio 1. E.g. if" + " scaleRange = 0.4 and scalesNumber = 2, then there will be 2 scales, 0.8 and 1.2."); +DEFINE_bool(hand_tracking, false, "Adding hand tracking might improve hand keypoints detection for webcam (if the frame rate" + " is high enough, i.e. >7 FPS per GPU) and video. This is not person ID tracking, it" + " simply looks for hands in positions at which hands were located in previous frames, but" + " it does not guarantee the same person ID among frames"); // OpenPose Rendering DEFINE_int32(part_to_show, 0, "Part to show from the start."); DEFINE_bool(disable_blending, false, "If blending is enabled, it will merge the results with the original frame. If disabled, it" " will only display the results."); // OpenPose Rendering Pose +DEFINE_double(render_threshold, 0.05, "Only estimated keypoints whose score confidences are higher than this threshold will be" + " rendered. Generally, a high threshold (> 0.5) will only render very clear body parts;" + " while small thresholds (~0.1) will also output guessed and occluded keypoints, but also" + " more false positives (i.e. wrong detections)."); DEFINE_int32(render_pose, 2, "Set to 0 for no rendering, 1 for CPU rendering (slightly faster), and 2 for GPU rendering" " (slower but greater functionality, e.g. `alpha_X` flags). If rendering is enabled, it will" " render both `outputData` and `cvOutputData` with the original image and desired body part" @@ -121,15 +127,17 @@ DEFINE_double(alpha_pose, 0.6, "Blending factor (range DEFINE_double(alpha_heatmap, 0.7, "Blending factor (range 0-1) between heatmap and original frame. 1 will only show the" " heatmap, 0 will only show the frame. Only valid for GPU rendering."); // OpenPose Rendering Face -DEFINE_int32(render_face, -1, "Analogous to `render_pose` but applied to the face. Extra option: -1 to use the same" +DEFINE_double(face_render_threshold, 0.4, "Analogous to `render_threshold`, but applied to the face keypoints."); +DEFINE_int32(face_render, -1, "Analogous to `render_pose` but applied to the face. Extra option: -1 to use the same" " configuration that `render_pose` is using."); -DEFINE_double(alpha_face, 0.6, "Analogous to `alpha_pose` but applied to face."); -DEFINE_double(alpha_heatmap_face, 0.7, "Analogous to `alpha_heatmap` but applied to face."); +DEFINE_double(face_alpha_pose, 0.6, "Analogous to `alpha_pose` but applied to face."); +DEFINE_double(face_alpha_heatmap, 0.7, "Analogous to `alpha_heatmap` but applied to face."); // OpenPose Rendering Hand -DEFINE_int32(render_hand, -1, "Analogous to `render_pose` but applied to the hand. Extra option: -1 to use the same" +DEFINE_double(hand_render_threshold, 0.2, "Analogous to `render_threshold`, but applied to the hand keypoints."); +DEFINE_int32(hand_render, -1, "Analogous to `render_pose` but applied to the hand. Extra option: -1 to use the same" " configuration that `render_pose` is using."); -DEFINE_double(alpha_hand, 0.6, "Analogous to `alpha_pose` but applied to hand."); -DEFINE_double(alpha_heatmap_hand, 0.7, "Analogous to `alpha_heatmap` but applied to hand."); +DEFINE_double(hand_alpha_pose, 0.6, "Analogous to `alpha_pose` but applied to hand."); +DEFINE_double(hand_alpha_heatmap, 0.7, "Analogous to `alpha_heatmap` but applied to hand."); // Display DEFINE_bool(fullscreen, false, "Run in full-screen mode (press f during runtime to toggle)."); DEFINE_bool(process_real_time, false, "Enable to keep the original source frame rate (e.g. for video). If the processing time is" @@ -187,18 +195,18 @@ int openPoseDemo() op::Wrapper> opWrapper; // Pose configuration (use WrapperStructPose{} for default and recommended configuration) const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, - FLAGS_num_gpu_start, FLAGS_num_scales, (float)FLAGS_scale_gap, + FLAGS_num_gpu_start, FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, op::ScaleMode::UnsignedChar}; + heatMapTypes, op::ScaleMode::UnsignedChar, (float)FLAGS_render_threshold}; // Face configuration (use op::WrapperStructFace{} to disable it) - const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_render_face, FLAGS_render_pose), - (float)FLAGS_alpha_face, (float)FLAGS_alpha_heatmap_face}; + const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), + (float)FLAGS_face_alpha_pose, (float)FLAGS_face_alpha_heatmap, (float)FLAGS_face_render_threshold}; // Hand configuration (use op::WrapperStructHand{} to disable it) - const op::WrapperStructHand wrapperStructHand{FLAGS_hand, handNetInputSize, op::flagsToDetectionMode(FLAGS_hand_detection_mode, producerSharedPtr), - op::flagsToRenderMode(FLAGS_render_hand, FLAGS_render_pose), (float)FLAGS_alpha_hand, - (float)FLAGS_alpha_heatmap_hand}; + const op::WrapperStructHand wrapperStructHand{FLAGS_hand, handNetInputSize, FLAGS_hand_scale_number, (float)FLAGS_hand_scale_range, + FLAGS_hand_tracking, op::flagsToRenderMode(FLAGS_hand_render, FLAGS_render_pose), + (float)FLAGS_hand_alpha_pose, (float)FLAGS_hand_alpha_heatmap, (float)FLAGS_hand_render_threshold}; // Producer (use default to disable any input) const op::WrapperStructInput wrapperStructInput{producerSharedPtr, FLAGS_frame_first, FLAGS_frame_last, FLAGS_process_real_time, FLAGS_frame_flip, FLAGS_frame_rotate, FLAGS_frames_repeat}; diff --git a/examples/tutorial_pose/1_extract_from_image.cpp b/examples/tutorial_pose/1_extract_from_image.cpp index 8f1a5d79..3ea6c2ed 100644 --- a/examples/tutorial_pose/1_extract_from_image.cpp +++ b/examples/tutorial_pose/1_extract_from_image.cpp @@ -35,13 +35,17 @@ DEFINE_string(net_resolution, "656x368", "Multiples of 16. If it DEFINE_string(resolution, "1280x720", "The image resolution (display and output). Use \"-1x-1\" to force the program to use the" " default images resolution."); DEFINE_int32(num_gpu_start, 0, "GPU device start number."); -DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless num_scales>1. Initial scale is always 1. If you" - " want to change the initial scale, you actually want to multiply the `net_resolution` by" - " your desired initial scale."); -DEFINE_int32(num_scales, 1, "Number of scales to average."); +DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." + " If you want to change the initial scale, you actually want to multiply the" + " `net_resolution` by your desired initial scale."); +DEFINE_int32(scale_number, 1, "Number of scales to average."); // OpenPose Rendering DEFINE_bool(disable_blending, false, "If blending is enabled, it will merge the results with the original frame. If disabled, it" " will only display the results."); +DEFINE_double(render_threshold, 0.05, "Only estimated keypoints whose score confidences are higher than this threshold will be" + " rendered. Generally, a high threshold (> 0.5) will only render very clear body parts;" + " while small thresholds (~0.1) will also output guessed and occluded keypoints, but also" + " more false positives (i.e. wrong detections)."); DEFINE_double(alpha_pose, 0.6, "Blending factor (range 0-1) for the body part rendering. 1 will show it completely, 0 will" " hide it. Only valid for GPU rendering."); @@ -67,16 +71,17 @@ int openPoseTutorialPose1() // Check no contradictory flags enabled if (FLAGS_alpha_pose < 0. || FLAGS_alpha_pose > 1.) op::error("Alpha value for blending must be in the range [0,1].", __LINE__, __FUNCTION__, __FILE__); - if (FLAGS_scale_gap <= 0. && FLAGS_num_scales > 1) - op::error("Incompatible flag configuration: scale_gap must be greater than 0 or num_scales = 1.", __LINE__, __FUNCTION__, __FILE__); + if (FLAGS_scale_gap <= 0. && FLAGS_scale_number > 1) + op::error("Incompatible flag configuration: scale_gap must be greater than 0 or scale_number = 1.", __LINE__, __FUNCTION__, __FILE__); // Logging op::log("", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Step 3 - Initialize all required classes - op::CvMatToOpInput cvMatToOpInput{netInputSize, FLAGS_num_scales, (float)FLAGS_scale_gap}; + op::CvMatToOpInput cvMatToOpInput{netInputSize, FLAGS_scale_number, (float)FLAGS_scale_gap}; op::CvMatToOpOutput cvMatToOpOutput{outputSize}; - op::PoseExtractorCaffe poseExtractorCaffe{netInputSize, netOutputSize, outputSize, FLAGS_num_scales, poseModel, + op::PoseExtractorCaffe poseExtractorCaffe{netInputSize, netOutputSize, outputSize, FLAGS_scale_number, poseModel, FLAGS_model_folder, FLAGS_num_gpu_start}; - op::PoseRenderer poseRenderer{netOutputSize, outputSize, poseModel, nullptr, !FLAGS_disable_blending, (float)FLAGS_alpha_pose}; + op::PoseRenderer poseRenderer{netOutputSize, outputSize, poseModel, nullptr, (float)FLAGS_render_threshold, + !FLAGS_disable_blending, (float)FLAGS_alpha_pose}; op::OpOutputToCvMat opOutputToCvMat{outputSize}; const op::Point windowedSize = outputSize; op::FrameDisplayer frameDisplayer{windowedSize, "OpenPose Tutorial - Example 1"}; diff --git a/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp b/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp index 1f142a70..52fc1a27 100644 --- a/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp +++ b/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp @@ -35,14 +35,18 @@ DEFINE_string(net_resolution, "656x368", "Multiples of 16. If it DEFINE_string(resolution, "1280x720", "The image resolution (display and output). Use \"-1x-1\" to force the program to use the" " default images resolution."); DEFINE_int32(num_gpu_start, 0, "GPU device start number."); -DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless num_scales>1. Initial scale is always 1. If you" - " want to change the initial scale, you actually want to multiply the `net_resolution` by" - " your desired initial scale."); -DEFINE_int32(num_scales, 1, "Number of scales to average."); +DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." + " If you want to change the initial scale, you actually want to multiply the" + " `net_resolution` by your desired initial scale."); +DEFINE_int32(scale_number, 1, "Number of scales to average."); // OpenPose Rendering DEFINE_int32(part_to_show, 19, "Part to show from the start."); DEFINE_bool(disable_blending, false, "If blending is enabled, it will merge the results with the original frame. If disabled, it" " will only display the results."); +DEFINE_double(render_threshold, 0.05, "Only estimated keypoints whose score confidences are higher than this threshold will be" + " rendered. Generally, a high threshold (> 0.5) will only render very clear body parts;" + " while small thresholds (~0.1) will also output guessed and occluded keypoints, but also" + " more false positives (i.e. wrong detections)."); DEFINE_double(alpha_pose, 0.6, "Blending factor (range 0-1) for the body part rendering. 1 will show it completely, 0 will" " hide it. Only valid for GPU rendering."); DEFINE_double(alpha_heatmap, 0.7, "Blending factor (range 0-1) between heatmap and original frame. 1 will only show the" @@ -70,18 +74,18 @@ int openPoseTutorialPose2() // Check no contradictory flags enabled if (FLAGS_alpha_pose < 0. || FLAGS_alpha_pose > 1.) op::error("Alpha value for blending must be in the range [0,1].", __LINE__, __FUNCTION__, __FILE__); - if (FLAGS_scale_gap <= 0. && FLAGS_num_scales > 1) - op::error("Incompatible flag configuration: scale_gap must be greater than 0 or num_scales = 1.", __LINE__, __FUNCTION__, __FILE__); + if (FLAGS_scale_gap <= 0. && FLAGS_scale_number > 1) + op::error("Incompatible flag configuration: scale_gap must be greater than 0 or scale_number = 1.", __LINE__, __FUNCTION__, __FILE__); // Logging op::log("", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Step 3 - Initialize all required classes - op::CvMatToOpInput cvMatToOpInput{netInputSize, FLAGS_num_scales, (float)FLAGS_scale_gap}; + op::CvMatToOpInput cvMatToOpInput{netInputSize, FLAGS_scale_number, (float)FLAGS_scale_gap}; op::CvMatToOpOutput cvMatToOpOutput{outputSize}; std::shared_ptr poseExtractorPtr = std::make_shared(netInputSize, netOutputSize, outputSize, - FLAGS_num_scales, poseModel, + FLAGS_scale_number, poseModel, FLAGS_model_folder, FLAGS_num_gpu_start); - op::PoseRenderer poseRenderer{netOutputSize, outputSize, poseModel, poseExtractorPtr, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, - (float)FLAGS_alpha_heatmap}; + op::PoseRenderer poseRenderer{netOutputSize, outputSize, poseModel, poseExtractorPtr, (float)FLAGS_render_threshold, + !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap}; poseRenderer.setElementToRender(FLAGS_part_to_show); op::OpOutputToCvMat opOutputToCvMat{outputSize}; const op::Point windowedSize = outputSize; diff --git a/examples/tutorial_wrapper/1_user_asynchronous.cpp b/examples/tutorial_wrapper/1_user_asynchronous.cpp index 89eaea0a..b475108f 100644 --- a/examples/tutorial_wrapper/1_user_asynchronous.cpp +++ b/examples/tutorial_wrapper/1_user_asynchronous.cpp @@ -58,15 +58,15 @@ DEFINE_int32(keypoint_scale, 0, "Scaling of the (x,y) co " Select `0` to scale it to the original source resolution, `1`to scale it to the net output" " size (set with `net_resolution`), `2` to scale it to the final output size (set with" " `resolution`), `3` to scale it in the range [0,1], and 4 for range [-1,1]. Non related" - " with `num_scales` and `scale_gap`."); + " with `scale_number` and `scale_gap`."); // OpenPose Body Pose DEFINE_string(model_pose, "COCO", "Model to be used (e.g. COCO, MPI, MPI_4_layers)."); DEFINE_string(net_resolution, "656x368", "Multiples of 16. If it is increased, the accuracy usually increases. If it is decreased," " the speed increases."); -DEFINE_int32(num_scales, 1, "Number of scales to average."); -DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless num_scales>1. Initial scale is always 1. If you" - " want to change the initial scale, you actually want to multiply the `net_resolution` by" - " your desired initial scale."); +DEFINE_int32(scale_number, 1, "Number of scales to average."); +DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." + " If you want to change the initial scale, you actually want to multiply the" + " `net_resolution` by your desired initial scale."); DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array" " (program speed will decrease). Not required for our library, enable it only if you intend" " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" @@ -87,18 +87,25 @@ DEFINE_string(face_net_resolution, "368x368", "Multiples of 16. Analog DEFINE_bool(hand, false, "Enables hand keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`."); DEFINE_string(hand_net_resolution, "368x368", "Multiples of 16. Analogous to `net_resolution` but applied to the hand keypoint detector."); -DEFINE_int32(hand_detection_mode, 0, "Set to 0 to perform 1-time keypoint detection (fastest), 1 for iterative detection" - " (recommended for images and fast videos, slow method), 2 for tracking (recommended for" - " webcam if the frame rate is >10 FPS per GPU used and for video, in practice as fast as" - " 1-time detection), 3 for both iterative and tracking (recommended for webcam if the" - " resulting frame rate is still >10 FPS and for video, ideally best result but slower), or" - " -1 (default) for automatic selection (fast method for webcam, tracking for video and" - " iterative for images)."); +DEFINE_int32(hand_scale_number, 1, "Analogous to `scale_number` but applied to the hand keypoint detector. Our best results" + " were found with `hand_scale_number` = 6 and `hand_scale_range` = 0.4"); +DEFINE_double(hand_scale_range, 0.4, "Analogous purpose than `scale_gap` but applied to the hand keypoint detector. Total range" + " between smallest and biggest scale. The scales will be centered in ratio 1. E.g. if" + " scaleRange = 0.4 and scalesNumber = 2, then there will be 2 scales, 0.8 and 1.2."); + +DEFINE_bool(hand_tracking, false, "Adding hand tracking might improve hand keypoints detection for webcam (if the frame rate" + " is high enough, i.e. >7 FPS per GPU) and video. This is not person ID tracking, it" + " simply looks for hands in positions at which hands were located in previous frames, but" + " it does not guarantee the same person ID among frames"); // OpenPose Rendering DEFINE_int32(part_to_show, 0, "Part to show from the start."); DEFINE_bool(disable_blending, false, "If blending is enabled, it will merge the results with the original frame. If disabled, it" " will only display the results."); // OpenPose Rendering Pose +DEFINE_double(render_threshold, 0.05, "Only estimated keypoints whose score confidences are higher than this threshold will be" + " rendered. Generally, a high threshold (> 0.5) will only render very clear body parts;" + " while small thresholds (~0.1) will also output guessed and occluded keypoints, but also" + " more false positives (i.e. wrong detections)."); DEFINE_int32(render_pose, 2, "Set to 0 for no rendering, 1 for CPU rendering (slightly faster), and 2 for GPU rendering" " (slower but greater functionality, e.g. `alpha_X` flags). If rendering is enabled, it will" " render both `outputData` and `cvOutputData` with the original image and desired body part" @@ -108,15 +115,17 @@ DEFINE_double(alpha_pose, 0.6, "Blending factor (range DEFINE_double(alpha_heatmap, 0.7, "Blending factor (range 0-1) between heatmap and original frame. 1 will only show the" " heatmap, 0 will only show the frame. Only valid for GPU rendering."); // OpenPose Rendering Face -DEFINE_int32(render_face, -1, "Analogous to `render_pose` but applied to the face. Extra option: -1 to use the same" +DEFINE_double(face_render_threshold, 0.4, "Analogous to `render_threshold`, but applied to the face keypoints."); +DEFINE_int32(face_render, -1, "Analogous to `render_pose` but applied to the face. Extra option: -1 to use the same" " configuration that `render_pose` is using."); -DEFINE_double(alpha_face, 0.6, "Analogous to `alpha_pose` but applied to face."); -DEFINE_double(alpha_heatmap_face, 0.7, "Analogous to `alpha_heatmap` but applied to face."); +DEFINE_double(face_alpha_pose, 0.6, "Analogous to `alpha_pose` but applied to face."); +DEFINE_double(face_alpha_heatmap, 0.7, "Analogous to `alpha_heatmap` but applied to face."); // OpenPose Rendering Hand -DEFINE_int32(render_hand, -1, "Analogous to `render_pose` but applied to the hand. Extra option: -1 to use the same" +DEFINE_double(hand_render_threshold, 0.2, "Analogous to `render_threshold`, but applied to the hand keypoints."); +DEFINE_int32(hand_render, -1, "Analogous to `render_pose` but applied to the hand. Extra option: -1 to use the same" " configuration that `render_pose` is using."); -DEFINE_double(alpha_hand, 0.6, "Analogous to `alpha_pose` but applied to hand."); -DEFINE_double(alpha_heatmap_hand, 0.7, "Analogous to `alpha_heatmap` but applied to hand."); +DEFINE_double(hand_alpha_pose, 0.6, "Analogous to `alpha_pose` but applied to hand."); +DEFINE_double(hand_alpha_heatmap, 0.7, "Analogous to `alpha_heatmap` but applied to hand."); // Result Saving DEFINE_string(write_images, "", "Directory to write rendered frames in `write_images_format` image format."); DEFINE_string(write_images_format, "png", "File extension and format for `write_images`, e.g. png, jpg or bmp. Check the OpenCV" @@ -260,18 +269,18 @@ int openPoseTutorialWrapper1() op::Wrapper> opWrapper{op::ThreadManagerMode::Asynchronous}; // Pose configuration (use WrapperStructPose{} for default and recommended configuration) const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, - FLAGS_num_gpu_start, FLAGS_num_scales, (float)FLAGS_scale_gap, + FLAGS_num_gpu_start, FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, heatMapScale}; + heatMapTypes, heatMapScale, (float)FLAGS_render_threshold}; // Face configuration (use op::WrapperStructFace{} to disable it) - const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_render_face, FLAGS_render_pose), - (float)FLAGS_alpha_face, (float)FLAGS_alpha_heatmap_face}; + const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), + (float)FLAGS_face_alpha_pose, (float)FLAGS_face_alpha_heatmap, (float)FLAGS_face_render_threshold}; // Hand configuration (use op::WrapperStructHand{} to disable it) - const op::WrapperStructHand wrapperStructHand{FLAGS_hand, handNetInputSize, op::flagsToDetectionMode(FLAGS_hand_detection_mode), - op::flagsToRenderMode(FLAGS_render_hand, FLAGS_render_pose), (float)FLAGS_alpha_hand, - (float)FLAGS_alpha_heatmap_hand}; + const op::WrapperStructHand wrapperStructHand{FLAGS_hand, handNetInputSize, FLAGS_hand_scale_number, (float)FLAGS_hand_scale_range, + FLAGS_hand_tracking, op::flagsToRenderMode(FLAGS_hand_render, FLAGS_render_pose), + (float)FLAGS_hand_alpha_pose, (float)FLAGS_hand_alpha_heatmap, (float)FLAGS_hand_render_threshold}; // Consumer (comment or use default argument to disable any output) const bool displayGui = false; const bool guiVerbose = false; diff --git a/examples/tutorial_wrapper/2_user_synchronous.cpp b/examples/tutorial_wrapper/2_user_synchronous.cpp index c1fd65c5..fbb26853 100644 --- a/examples/tutorial_wrapper/2_user_synchronous.cpp +++ b/examples/tutorial_wrapper/2_user_synchronous.cpp @@ -58,15 +58,15 @@ DEFINE_int32(keypoint_scale, 0, "Scaling of the (x,y) co " Select `0` to scale it to the original source resolution, `1`to scale it to the net output" " size (set with `net_resolution`), `2` to scale it to the final output size (set with" " `resolution`), `3` to scale it in the range [0,1], and 4 for range [-1,1]. Non related" - " with `num_scales` and `scale_gap`."); + " with `scale_number` and `scale_gap`."); // OpenPose Body Pose DEFINE_string(model_pose, "COCO", "Model to be used (e.g. COCO, MPI, MPI_4_layers)."); DEFINE_string(net_resolution, "656x368", "Multiples of 16. If it is increased, the accuracy usually increases. If it is decreased," " the speed increases."); -DEFINE_int32(num_scales, 1, "Number of scales to average."); -DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless num_scales>1. Initial scale is always 1. If you" - " want to change the initial scale, you actually want to multiply the `net_resolution` by" - " your desired initial scale."); +DEFINE_int32(scale_number, 1, "Number of scales to average."); +DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." + " If you want to change the initial scale, you actually want to multiply the" + " `net_resolution` by your desired initial scale."); DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array" " (program speed will decrease). Not required for our library, enable it only if you intend" " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" @@ -87,18 +87,25 @@ DEFINE_string(face_net_resolution, "368x368", "Multiples of 16. Analog DEFINE_bool(hand, false, "Enables hand keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`."); DEFINE_string(hand_net_resolution, "368x368", "Multiples of 16. Analogous to `net_resolution` but applied to the hand keypoint detector."); -DEFINE_int32(hand_detection_mode, 0, "Set to 0 to perform 1-time keypoint detection (fastest), 1 for iterative detection" - " (recommended for images and fast videos, slow method), 2 for tracking (recommended for" - " webcam if the frame rate is >10 FPS per GPU used and for video, in practice as fast as" - " 1-time detection), 3 for both iterative and tracking (recommended for webcam if the" - " resulting frame rate is still >10 FPS and for video, ideally best result but slower), or" - " -1 (default) for automatic selection (fast method for webcam, tracking for video and" - " iterative for images)."); +DEFINE_int32(hand_scale_number, 1, "Analogous to `scale_number` but applied to the hand keypoint detector. Our best results" + " were found with `hand_scale_number` = 6 and `hand_scale_range` = 0.4"); +DEFINE_double(hand_scale_range, 0.4, "Analogous purpose than `scale_gap` but applied to the hand keypoint detector. Total range" + " between smallest and biggest scale. The scales will be centered in ratio 1. E.g. if" + " scaleRange = 0.4 and scalesNumber = 2, then there will be 2 scales, 0.8 and 1.2."); + +DEFINE_bool(hand_tracking, false, "Adding hand tracking might improve hand keypoints detection for webcam (if the frame rate" + " is high enough, i.e. >7 FPS per GPU) and video. This is not person ID tracking, it" + " simply looks for hands in positions at which hands were located in previous frames, but" + " it does not guarantee the same person ID among frames"); // OpenPose Rendering DEFINE_int32(part_to_show, 0, "Part to show from the start."); DEFINE_bool(disable_blending, false, "If blending is enabled, it will merge the results with the original frame. If disabled, it" " will only display the results."); // OpenPose Rendering Pose +DEFINE_double(render_threshold, 0.05, "Only estimated keypoints whose score confidences are higher than this threshold will be" + " rendered. Generally, a high threshold (> 0.5) will only render very clear body parts;" + " while small thresholds (~0.1) will also output guessed and occluded keypoints, but also" + " more false positives (i.e. wrong detections)."); DEFINE_int32(render_pose, 2, "Set to 0 for no rendering, 1 for CPU rendering (slightly faster), and 2 for GPU rendering" " (slower but greater functionality, e.g. `alpha_X` flags). If rendering is enabled, it will" " render both `outputData` and `cvOutputData` with the original image and desired body part" @@ -108,15 +115,17 @@ DEFINE_double(alpha_pose, 0.6, "Blending factor (range DEFINE_double(alpha_heatmap, 0.7, "Blending factor (range 0-1) between heatmap and original frame. 1 will only show the" " heatmap, 0 will only show the frame. Only valid for GPU rendering."); // OpenPose Rendering Face -DEFINE_int32(render_face, -1, "Analogous to `render_pose` but applied to the face. Extra option: -1 to use the same" +DEFINE_double(face_render_threshold, 0.4, "Analogous to `render_threshold`, but applied to the face keypoints."); +DEFINE_int32(face_render, -1, "Analogous to `render_pose` but applied to the face. Extra option: -1 to use the same" " configuration that `render_pose` is using."); -DEFINE_double(alpha_face, 0.6, "Analogous to `alpha_pose` but applied to face."); -DEFINE_double(alpha_heatmap_face, 0.7, "Analogous to `alpha_heatmap` but applied to face."); +DEFINE_double(face_alpha_pose, 0.6, "Analogous to `alpha_pose` but applied to face."); +DEFINE_double(face_alpha_heatmap, 0.7, "Analogous to `alpha_heatmap` but applied to face."); // OpenPose Rendering Hand -DEFINE_int32(render_hand, -1, "Analogous to `render_pose` but applied to the hand. Extra option: -1 to use the same" +DEFINE_double(hand_render_threshold, 0.2, "Analogous to `render_threshold`, but applied to the hand keypoints."); +DEFINE_int32(hand_render, -1, "Analogous to `render_pose` but applied to the hand. Extra option: -1 to use the same" " configuration that `render_pose` is using."); -DEFINE_double(alpha_hand, 0.6, "Analogous to `alpha_pose` but applied to hand."); -DEFINE_double(alpha_heatmap_hand, 0.7, "Analogous to `alpha_heatmap` but applied to hand."); +DEFINE_double(hand_alpha_pose, 0.6, "Analogous to `alpha_pose` but applied to hand."); +DEFINE_double(hand_alpha_heatmap, 0.7, "Analogous to `alpha_heatmap` but applied to hand."); // Result Saving DEFINE_string(write_images, "", "Directory to write rendered frames in `write_images_format` image format."); DEFINE_string(write_images_format, "png", "File extension and format for `write_images`, e.g. png, jpg or bmp. Check the OpenCV" @@ -321,18 +330,18 @@ int openPoseTutorialWrapper2() opWrapper.setWorkerOutput(wUserOutput, workerOutputOnNewThread); // Configure OpenPose const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, - FLAGS_num_gpu_start, FLAGS_num_scales, (float)FLAGS_scale_gap, + FLAGS_num_gpu_start, FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, heatMapScale}; + heatMapTypes, heatMapScale, (float)FLAGS_render_threshold}; // Face configuration (use op::WrapperStructFace{} to disable it) - const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_render_face, FLAGS_render_pose), - (float)FLAGS_alpha_face, (float)FLAGS_alpha_heatmap_face}; + const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), + (float)FLAGS_face_alpha_pose, (float)FLAGS_face_alpha_heatmap, (float)FLAGS_face_render_threshold}; // Hand configuration (use op::WrapperStructHand{} to disable it) - const op::WrapperStructHand wrapperStructHand{FLAGS_hand, handNetInputSize, op::flagsToDetectionMode(FLAGS_hand_detection_mode), - op::flagsToRenderMode(FLAGS_render_hand, FLAGS_render_pose), (float)FLAGS_alpha_hand, - (float)FLAGS_alpha_heatmap_hand}; + const op::WrapperStructHand wrapperStructHand{FLAGS_hand, handNetInputSize, FLAGS_hand_scale_number, (float)FLAGS_hand_scale_range, + FLAGS_hand_tracking, op::flagsToRenderMode(FLAGS_hand_render, FLAGS_render_pose), + (float)FLAGS_hand_alpha_pose, (float)FLAGS_hand_alpha_heatmap, (float)FLAGS_hand_render_threshold}; // Consumer (comment or use default argument to disable any output) const bool displayGui = false; const bool guiVerbose = false; diff --git a/include/openpose/core/datum.hpp b/include/openpose/core/datum.hpp index b24c512a..e49d5761 100644 --- a/include/openpose/core/datum.hpp +++ b/include/openpose/core/datum.hpp @@ -107,7 +107,7 @@ namespace op float scaleNetToOutput; /**< Scale ratio between the net output and the final output Datum::cvOutputData. */ - std::vector scaleRatios; /**< Scale ratios between each scale (e.g. flag `num_scales`). Used to resize the different scales. */ + std::vector scaleRatios; /**< Scale ratios between each scale (e.g. flag `scale_number`). Used to resize the different scales. */ std::pair elementRendered; /**< Pair with the element key id POSE_BODY_PART_MAPPING on `pose/poseParameters.hpp` and its mapped value (e.g. 1 and "Neck"). */ diff --git a/include/openpose/core/enumClasses.hpp b/include/openpose/core/enumClasses.hpp index efc4df41..ea1e0960 100644 --- a/include/openpose/core/enumClasses.hpp +++ b/include/openpose/core/enumClasses.hpp @@ -20,14 +20,6 @@ namespace op PAFs, }; - enum class DetectionMode : unsigned char - { - Fast, - Iterative, - Tracking, - IterativeAndTracking, - }; - enum class RenderMode : unsigned char { None, diff --git a/include/openpose/face/faceParameters.hpp b/include/openpose/face/faceParameters.hpp index 753e2f8b..20405924 100644 --- a/include/openpose/face/faceParameters.hpp +++ b/include/openpose/face/faceParameters.hpp @@ -23,7 +23,6 @@ namespace op // Rendering parameters const auto FACE_DEFAULT_ALPHA_KEYPOINT = POSE_DEFAULT_ALPHA_KEYPOINT; const auto FACE_DEFAULT_ALPHA_HEAT_MAP = POSE_DEFAULT_ALPHA_HEAT_MAP; - const auto FACE_RENDER_THRESHOLD = 0.4f; } #endif // OPENPOSE_FACE_FACE_PARAMETERS_HPP diff --git a/include/openpose/face/faceRenderer.hpp b/include/openpose/face/faceRenderer.hpp index 02038063..4af16257 100644 --- a/include/openpose/face/faceRenderer.hpp +++ b/include/openpose/face/faceRenderer.hpp @@ -13,8 +13,10 @@ namespace op class FaceRenderer : public Renderer { public: - explicit FaceRenderer(const Point& frameSize, const float alphaKeypoint = FACE_DEFAULT_ALPHA_KEYPOINT, - const float alphaHeatMap = FACE_DEFAULT_ALPHA_HEAT_MAP, const RenderMode renderMode = RenderMode::Cpu); + FaceRenderer(const Point& frameSize, const float renderThreshold, + const float alphaKeypoint = FACE_DEFAULT_ALPHA_KEYPOINT, + const float alphaHeatMap = FACE_DEFAULT_ALPHA_HEAT_MAP, + const RenderMode renderMode = RenderMode::Cpu); ~FaceRenderer(); @@ -23,6 +25,7 @@ namespace op void renderFace(Array& outputData, const Array& faceKeypoints); private: + const float mRenderThreshold; const Point mFrameSize; const RenderMode mRenderMode; float* pGpuFace; // GPU aux memory diff --git a/include/openpose/face/renderFace.hpp b/include/openpose/face/renderFace.hpp index e1e18039..a490b639 100644 --- a/include/openpose/face/renderFace.hpp +++ b/include/openpose/face/renderFace.hpp @@ -7,10 +7,10 @@ namespace op { - void renderFaceKeypointsCpu(Array& frameArray, const Array& faceKeypoints); + void renderFaceKeypointsCpu(Array& frameArray, const Array& faceKeypoints, const float renderThreshold); - void renderFaceKeypointsGpu(float* framePtr, const Point& frameSize, const float* const facePtr, const int numberFace, - const float alphaColorToAdd = FACE_DEFAULT_ALPHA_KEYPOINT); + void renderFaceKeypointsGpu(float* framePtr, const Point& frameSize, const float* const facePtr, const int numberPeople, + const float renderThreshold, const float alphaColorToAdd = FACE_DEFAULT_ALPHA_KEYPOINT); } #endif // OPENPOSE_FACE_RENDER_FACE_HPP diff --git a/include/openpose/hand/handExtractor.hpp b/include/openpose/hand/handExtractor.hpp index 25f420ea..4593fac7 100644 --- a/include/openpose/hand/handExtractor.hpp +++ b/include/openpose/hand/handExtractor.hpp @@ -20,7 +20,7 @@ namespace op { public: explicit HandExtractor(const Point& netInputSize, const Point& netOutputSize, const std::string& modelFolder, const int gpuId, - const bool iterativeDetection = false); + const unsigned short numberScales = 1, const float rangeScales = 0.4f); void initializationOnThread(); @@ -30,7 +30,8 @@ namespace op std::array, 2> getHandKeypoints() const; private: - const bool mIterativeDetection; + // const bool mMultiScaleDetection; + const std::pair mMultiScaleNumberAndRange; const Point mNetOutputSize; std::shared_ptr spNet; std::shared_ptr> spResizeAndMergeCaffe; diff --git a/include/openpose/hand/handParameters.hpp b/include/openpose/hand/handParameters.hpp index edc95fcf..56e89d55 100644 --- a/include/openpose/hand/handParameters.hpp +++ b/include/openpose/hand/handParameters.hpp @@ -5,7 +5,7 @@ namespace op { - const auto HAND_MAX_HANDS = POSE_MAX_PEOPLE; + const auto HAND_MAX_HANDS = 2*POSE_MAX_PEOPLE; const auto HAND_NUMBER_PARTS = 21u; #define HAND_PAIRS_RENDER_GPU {0,1, 1,2, 2,3, 3,4, 0,5, 5,6, 6,7, 7,8, 0,9, 9,10, 10,11, 11,12, 0,13, 13,14, 14,15, 15,16, 0,17, 17,18, 18,19, 19,20} @@ -36,12 +36,11 @@ namespace op // Constant parameters const auto HAND_CCN_DECREASE_FACTOR = 8.f; const std::string HAND_PROTOTXT{"hand/pose_deploy.prototxt"}; - const std::string HAND_TRAINED_MODEL{"hand/pose_iter_120000.caffemodel"}; + const std::string HAND_TRAINED_MODEL{"hand/pose_iter_102000.caffemodel"}; // Rendering parameters const auto HAND_DEFAULT_ALPHA_KEYPOINT = POSE_DEFAULT_ALPHA_KEYPOINT; const auto HAND_DEFAULT_ALPHA_HEAT_MAP = POSE_DEFAULT_ALPHA_HEAT_MAP; - const auto HAND_RENDER_THRESHOLD = 0.05f; } #endif // OPENPOSE_HAND_HAND_PARAMETERS_HPP diff --git a/include/openpose/hand/handRenderer.hpp b/include/openpose/hand/handRenderer.hpp index 1087214a..ae0bae21 100644 --- a/include/openpose/hand/handRenderer.hpp +++ b/include/openpose/hand/handRenderer.hpp @@ -14,8 +14,10 @@ namespace op class HandRenderer : public Renderer { public: - HandRenderer(const Point& frameSize, const float alphaKeypoint = HAND_DEFAULT_ALPHA_KEYPOINT, - const float alphaHeatMap = HAND_DEFAULT_ALPHA_HEAT_MAP, const RenderMode renderMode = RenderMode::Cpu); + HandRenderer(const Point& frameSize, const float renderThreshold, + const float alphaKeypoint = HAND_DEFAULT_ALPHA_KEYPOINT, + const float alphaHeatMap = HAND_DEFAULT_ALPHA_HEAT_MAP, + const RenderMode renderMode = RenderMode::Cpu); ~HandRenderer(); @@ -24,6 +26,7 @@ namespace op void renderHand(Array& outputData, const std::array, 2>& handKeypoints); private: + const float mRenderThreshold; const Point mFrameSize; const RenderMode mRenderMode; float* pGpuHand; // GPU aux memory diff --git a/include/openpose/hand/renderHand.hpp b/include/openpose/hand/renderHand.hpp index cf5bdfc7..c3c2eb94 100644 --- a/include/openpose/hand/renderHand.hpp +++ b/include/openpose/hand/renderHand.hpp @@ -8,10 +8,12 @@ namespace op { - void renderHandKeypointsCpu(Array& frameArray, const std::array, 2>& handKeypoints); + void renderHandKeypointsCpu(Array& frameArray, const std::array, 2>& handKeypoints, + const float renderThreshold); - void renderHandKeypointsGpu(float* framePtr, const Point& frameSize, const float* const handsPtr, const int numberHands, - const float alphaColorToAdd = HAND_DEFAULT_ALPHA_KEYPOINT); + void renderHandKeypointsGpu(float* framePtr, const Point& frameSize, const float* const handsPtr, + const int numberHands, const float renderThreshold, + const float alphaColorToAdd = HAND_DEFAULT_ALPHA_KEYPOINT); } #endif // OPENPOSE_HAND_GPU_HAND_RENDER_HPP diff --git a/include/openpose/pose/poseParameters.hpp b/include/openpose/pose/poseParameters.hpp index d81b0b69..179ea724 100644 --- a/include/openpose/pose/poseParameters.hpp +++ b/include/openpose/pose/poseParameters.hpp @@ -210,7 +210,6 @@ namespace op // Rendering parameters const auto POSE_DEFAULT_ALPHA_KEYPOINT = 0.6f; const auto POSE_DEFAULT_ALPHA_HEAT_MAP = 0.7f; - const auto POSE_RENDER_THRESHOLD = 0.01f; // Auxiliary functions unsigned int poseBodyPartMapStringToKey(const PoseModel poseModel, const std::string& string); diff --git a/include/openpose/pose/poseRenderer.hpp b/include/openpose/pose/poseRenderer.hpp index daf6c57a..cfc7c85b 100644 --- a/include/openpose/pose/poseRenderer.hpp +++ b/include/openpose/pose/poseRenderer.hpp @@ -15,10 +15,11 @@ namespace op class PoseRenderer : public Renderer { public: - explicit PoseRenderer(const Point& heatMapsSize, const Point& outputSize, const PoseModel poseModel, - const std::shared_ptr& poseExtractor, const bool blendOriginalFrame = true, - const float alphaKeypoint = POSE_DEFAULT_ALPHA_KEYPOINT, const float alphaHeatMap = POSE_DEFAULT_ALPHA_HEAT_MAP, - const unsigned int elementToRender = 0u, const RenderMode renderMode = RenderMode::Cpu); + PoseRenderer(const Point& heatMapsSize, const Point& outputSize, const PoseModel poseModel, + const std::shared_ptr& poseExtractor, const float renderThreshold, + const bool blendOriginalFrame = true, const float alphaKeypoint = POSE_DEFAULT_ALPHA_KEYPOINT, + const float alphaHeatMap = POSE_DEFAULT_ALPHA_HEAT_MAP, const unsigned int elementToRender = 0u, + const RenderMode renderMode = RenderMode::Cpu); ~PoseRenderer(); @@ -35,6 +36,7 @@ namespace op std::pair renderPose(Array& outputData, const Array& poseKeypoints, const float scaleNetToOutput = -1.f); private: + const float mRenderThreshold; const Point mHeatMapsSize; const Point mOutputSize; const PoseModel mPoseModel; diff --git a/include/openpose/pose/renderPose.hpp b/include/openpose/pose/renderPose.hpp index d8e86c2f..b9e7ad77 100644 --- a/include/openpose/pose/renderPose.hpp +++ b/include/openpose/pose/renderPose.hpp @@ -10,26 +10,27 @@ namespace op { void renderPoseKeypointsCpu(Array& frameArray, const Array& poseKeypoints, const PoseModel poseModel, - const bool blendOriginalFrame = true); + const float renderThreshold, const bool blendOriginalFrame = true); - void renderPoseKeypointsGpu(float* framePtr, const PoseModel poseModel, const int numberPeople, const Point& frameSize, - const float* const posePtr, const bool googlyEyes = false, const bool blendOriginalFrame = true, + void renderPoseKeypointsGpu(float* framePtr, const PoseModel poseModel, const int numberPeople, + const Point& frameSize, const float* const posePtr, const float renderThreshold, + const bool googlyEyes = false, const bool blendOriginalFrame = true, const float alphaBlending = POSE_DEFAULT_ALPHA_KEYPOINT); - void renderPoseHeatMapGpu(float* frame, const PoseModel poseModel, const Point& frameSize, const float* const heatmap, - const Point& heatmapSize, const float scaleToKeepRatio, const int part, - const float alphaBlending = POSE_DEFAULT_ALPHA_HEAT_MAP); + void renderPoseHeatMapGpu(float* frame, const PoseModel poseModel, const Point& frameSize, + const float* const heatmap, const Point& heatmapSize, const float scaleToKeepRatio, + const int part, const float alphaBlending = POSE_DEFAULT_ALPHA_HEAT_MAP); - void renderPoseHeatMapsGpu(float* frame, const PoseModel poseModel, const Point& frameSize, const float* const heatmap, - const Point& heatmapSize, const float scaleToKeepRatio, + void renderPoseHeatMapsGpu(float* frame, const PoseModel poseModel, const Point& frameSize, + const float* const heatmap, const Point& heatmapSize, const float scaleToKeepRatio, const float alphaBlending = POSE_DEFAULT_ALPHA_HEAT_MAP); - void renderPosePAFGpu(float* framePtr, const PoseModel poseModel, const Point& frameSize, const float* const heatmapPtr, - const Point& heatmapSize, const float scaleToKeepRatio, const int part, - const float alphaBlending = POSE_DEFAULT_ALPHA_HEAT_MAP); + void renderPosePAFGpu(float* framePtr, const PoseModel poseModel, const Point& frameSize, + const float* const heatmapPtr, const Point& heatmapSize, const float scaleToKeepRatio, + const int part, const float alphaBlending = POSE_DEFAULT_ALPHA_HEAT_MAP); - void renderPosePAFsGpu(float* framePtr, const PoseModel poseModel, const Point& frameSize, const float* const heatmapPtr, - const Point& heatmapSize, const float scaleToKeepRatio, + void renderPosePAFsGpu(float* framePtr, const PoseModel poseModel, const Point& frameSize, + const float* const heatmapPtr, const Point& heatmapSize, const float scaleToKeepRatio, const float alphaBlending = POSE_DEFAULT_ALPHA_HEAT_MAP); } diff --git a/include/openpose/utilities/flagsToOpenPose.hpp b/include/openpose/utilities/flagsToOpenPose.hpp index 8d0b0609..ec6d42d7 100644 --- a/include/openpose/utilities/flagsToOpenPose.hpp +++ b/include/openpose/utilities/flagsToOpenPose.hpp @@ -25,8 +25,6 @@ namespace op std::vector flagsToHeatMaps(const bool heatMapsAddParts = false, const bool heatMapsAddBkg = false, const bool heatMapsAddPAFs = false); - DetectionMode flagsToDetectionMode(const int handDetectionModeFlag, const std::shared_ptr& producer = nullptr); - RenderMode flagsToRenderMode(const int renderFlag, const int renderPoseFlag = -2); Point flagsToPoint(const std::string& pointString, const std::string& pointExample = "1280x720"); diff --git a/include/openpose/wrapper/wrapper.hpp b/include/openpose/wrapper/wrapper.hpp index bab9f94a..768180db 100644 --- a/include/openpose/wrapper/wrapper.hpp +++ b/include/openpose/wrapper/wrapper.hpp @@ -593,7 +593,7 @@ namespace op { poseRenderers.emplace_back(std::make_shared( poseNetOutputSize, finalOutputSize, wrapperStructPose.poseModel, poseExtractors[gpuId], - wrapperStructPose.blendOriginalFrame, alphaKeypoint, + wrapperStructPose.renderThreshold, wrapperStructPose.blendOriginalFrame, alphaKeypoint, alphaHeatMap, wrapperStructPose.defaultPartToRender, wrapperStructPose.renderMode )); } @@ -603,7 +603,7 @@ namespace op { poseCpuRenderer = std::make_shared( poseNetOutputSize, finalOutputSize, wrapperStructPose.poseModel, nullptr, - wrapperStructPose.blendOriginalFrame, alphaKeypoint, + wrapperStructPose.renderThreshold, wrapperStructPose.blendOriginalFrame, alphaKeypoint, alphaHeatMap, wrapperStructPose.defaultPartToRender, wrapperStructPose.renderMode ); cpuRenderers.emplace_back(std::make_shared>(poseCpuRenderer)); @@ -649,8 +649,7 @@ namespace op { // Hand detector // If tracking - if (wrapperStructHand.detectionMode == DetectionMode::Tracking - || wrapperStructHand.detectionMode == DetectionMode::IterativeAndTracking) + if (wrapperStructHand.tracking) spWPoses.at(gpuId).emplace_back(std::make_shared>(handDetector)); // If detection else @@ -658,14 +657,12 @@ namespace op // Hand keypoint extractor const auto netOutputSize = wrapperStructHand.netInputSize; const auto handExtractor = std::make_shared( - wrapperStructHand.netInputSize, netOutputSize, wrapperStructPose.modelFolder, gpuId + gpuNumberStart, - (wrapperStructHand.detectionMode == DetectionMode::Iterative - || wrapperStructHand.detectionMode == DetectionMode::IterativeAndTracking) + wrapperStructHand.netInputSize, netOutputSize, wrapperStructPose.modelFolder, + gpuId + gpuNumberStart, wrapperStructHand.scalesNumber, wrapperStructHand.scaleRange ); spWPoses.at(gpuId).emplace_back(std::make_shared>(handExtractor)); // If tracking - if (wrapperStructHand.detectionMode == DetectionMode::Tracking - || wrapperStructHand.detectionMode == DetectionMode::IterativeAndTracking) + if (wrapperStructHand.tracking) spWPoses.at(gpuId).emplace_back(std::make_shared>(handDetector)); } } @@ -682,7 +679,8 @@ namespace op if (wrapperStructFace.renderMode == RenderMode::Cpu) { // Construct face renderer - const auto faceRenderer = std::make_shared(finalOutputSize, wrapperStructFace.alphaKeypoint, + const auto faceRenderer = std::make_shared(finalOutputSize, wrapperStructFace.renderThreshold, + wrapperStructFace.alphaKeypoint, wrapperStructFace.alphaHeatMap, wrapperStructFace.renderMode); // Add worker @@ -694,7 +692,8 @@ namespace op for (auto i = 0; i < spWPoses.size(); i++) { // Construct face renderer - const auto faceRenderer = std::make_shared(finalOutputSize, wrapperStructFace.alphaKeypoint, + const auto faceRenderer = std::make_shared(finalOutputSize, wrapperStructFace.renderThreshold, + wrapperStructFace.alphaKeypoint, wrapperStructFace.alphaHeatMap, wrapperStructFace.renderMode); // Performance boost -> share spGpuMemoryPtr for all renderers @@ -718,7 +717,8 @@ namespace op if (wrapperStructHand.renderMode == RenderMode::Cpu) { // Construct hand renderer - const auto handRenderer = std::make_shared(finalOutputSize, wrapperStructHand.alphaKeypoint, + const auto handRenderer = std::make_shared(finalOutputSize, wrapperStructHand.renderThreshold, + wrapperStructHand.alphaKeypoint, wrapperStructHand.alphaHeatMap, wrapperStructHand.renderMode); // Add worker @@ -730,7 +730,8 @@ namespace op for (auto i = 0; i < spWPoses.size(); i++) { // Construct hands renderer - const auto handRenderer = std::make_shared(finalOutputSize, wrapperStructHand.alphaKeypoint, + const auto handRenderer = std::make_shared(finalOutputSize, wrapperStructHand.renderThreshold, + wrapperStructHand.alphaKeypoint, wrapperStructHand.alphaHeatMap, wrapperStructHand.renderMode); // Performance boost -> share spGpuMemoryPtr for all renderers diff --git a/include/openpose/wrapper/wrapperStructFace.hpp b/include/openpose/wrapper/wrapperStructFace.hpp index 8a4b2736..87cf0ac8 100644 --- a/include/openpose/wrapper/wrapperStructFace.hpp +++ b/include/openpose/wrapper/wrapperStructFace.hpp @@ -44,6 +44,13 @@ namespace op */ float alphaHeatMap; + /** + * Rendering threshold. Only estimated keypoints whose score confidences are higher than this value will be rendered. Generally, a + * high threshold (> 0.5) will only render very clear body parts; while small thresholds (~0.1) will also output guessed and occluded + * keypoints, but also more false positives (i.e. wrong detections). + */ + float renderThreshold; + /** * Constructor of the struct. * It has the recommended and default values we recommend for each element of the struct. @@ -52,7 +59,8 @@ namespace op WrapperStructFace(const bool enable = false, const Point& netInputSize = Point{368, 368}, const RenderMode renderMode = RenderMode::None, const float alphaKeypoint = FACE_DEFAULT_ALPHA_KEYPOINT, - const float alphaHeatMap = FACE_DEFAULT_ALPHA_HEAT_MAP); + const float alphaHeatMap = FACE_DEFAULT_ALPHA_HEAT_MAP, + const float renderThreshold = 0.4f); }; } diff --git a/include/openpose/wrapper/wrapperStructHand.hpp b/include/openpose/wrapper/wrapperStructHand.hpp index dee9fed3..b8944893 100644 --- a/include/openpose/wrapper/wrapperStructHand.hpp +++ b/include/openpose/wrapper/wrapperStructHand.hpp @@ -27,10 +27,24 @@ namespace op Point netInputSize; /** - * Whether to perform 1-time keypoint detection (fastest), iterative detection (recommended for images and fast videos, slowest method), or - * tracking (recommended for video and webcam, in practice as fast as 1-time detection). + * Number of scales to process. + * The greater, the slower and more memory it will be needed, but it will potentially increase accuracy. + * This parameter is related with scaleRange, such as the final pose estimation will be an average of the predicted results for each scale. */ - DetectionMode detectionMode; + int scalesNumber; + + /** + * Total range between smallest and biggest scale. The scales will be centered in ratio 1. E.g. if scaleRange = 0.4 and + * scalesNumber = 2, then there will be 2 scales, 0.8 and 1.2. + */ + float scaleRange; + + /** + * Whether to add tracking between frames. Adding hand tracking might improve hand keypoints detection for webcam (if the frame rate + * is high enough, i.e. >7 FPS per GPU) and video. This is not person ID tracking, it simply looks for hands in positions at which hands + * were located in previous frames, but it does not guarantee the same person id among frames. + */ + bool tracking; /** * Whether to render the output (pose locations, body, background or PAF heat maps) with CPU or GPU. @@ -50,16 +64,24 @@ namespace op */ float alphaHeatMap; + /** + * Rendering threshold. Only estimated keypoints whose score confidences are higher than this value will be rendered. Generally, a + * high threshold (> 0.5) will only render very clear body parts; while small thresholds (~0.1) will also output guessed and occluded + * keypoints, but also more false positives (i.e. wrong detections). + */ + float renderThreshold; + /** * Constructor of the struct. * It has the recommended and default values we recommend for each element of the struct. * Since all the elements of the struct are public, they can also be manually filled. */ WrapperStructHand(const bool enable = false, const Point& netInputSize = Point{368, 368}, - const DetectionMode detectionMode = DetectionMode::Fast, - const RenderMode renderMode = RenderMode::None, + const int scalesNumber = 1, const float scaleRange = 0.4f, + const bool tracking = false, const RenderMode renderMode = RenderMode::None, const float alphaKeypoint = HAND_DEFAULT_ALPHA_KEYPOINT, - const float alphaHeatMap = HAND_DEFAULT_ALPHA_HEAT_MAP); + const float alphaHeatMap = HAND_DEFAULT_ALPHA_HEAT_MAP, + const float renderThreshold = 0.2f); }; } diff --git a/include/openpose/wrapper/wrapperStructPose.hpp b/include/openpose/wrapper/wrapperStructPose.hpp index 634162fc..1096ad13 100644 --- a/include/openpose/wrapper/wrapperStructPose.hpp +++ b/include/openpose/wrapper/wrapperStructPose.hpp @@ -118,6 +118,13 @@ namespace op */ ScaleMode heatMapScale; + /** + * Rendering threshold. Only estimated keypoints whose score confidences are higher than this value will be rendered. Generally, a + * high threshold (> 0.5) will only render very clear body parts; while small thresholds (~0.1) will also output guessed and occluded + * keypoints, but also more false positives (i.e. wrong detections). + */ + float renderThreshold; + /** * Constructor of the struct. * It has the recommended and default values we recommend for each element of the struct. @@ -133,7 +140,8 @@ namespace op const float alphaHeatMap = POSE_DEFAULT_ALPHA_HEAT_MAP, const int defaultPartToRender = 0, const std::string& modelFolder = "models/", const std::vector& heatMapTypes = {}, - const ScaleMode heatMapScale = ScaleMode::ZeroToOne); + const ScaleMode heatMapScale = ScaleMode::ZeroToOne, + const float renderThreshold = 0.05f); }; } diff --git a/models/getModels.bat b/models/getModels.bat index 4d9577e4..6b559c41 100644 --- a/models/getModels.bat +++ b/models/getModels.bat @@ -7,6 +7,7 @@ SET WGET_EXE=..\3rdparty\windows\wget\wget.exe SET OPENPOSE_URL=http://posefs1.perception.cs.cmu.edu/OpenPose/models/ SET POSE_FOLDER=pose/ SET FACE_FOLDER=face/ +SET HAND_FOLDER=hand/ echo: echo ------------------------- POSE MODELS ------------------------- @@ -28,3 +29,10 @@ echo Face SET FACE_MODEL=%FACE_FOLDER%pose_iter_116000.caffemodel %WGET_EXE% -c http://posefs1.perception.cs.cmu.edu/OpenPose/models/%FACE_MODEL% -P %FACE_FOLDER% echo ----------------------- FACE DOWNLOADED ----------------------- + +echo: +echo ------------------------- HAND MODELS ------------------------- +echo Hand +SET HAND_MODEL=%HAND_FOLDER%pose_iter_102000.caffemodel +%WGET_EXE% -c http://posefs1.perception.cs.cmu.edu/OpenPose/models/%HAND_MODEL% -P %HAND_FOLDER% +echo ----------------------- HAND DOWNLOADED ----------------------- diff --git a/models/getModels.sh b/models/getModels.sh index cf6bd00e..5a6c94c4 100755 --- a/models/getModels.sh +++ b/models/getModels.sh @@ -3,6 +3,7 @@ OPENPOSE_URL="http://posefs1.perception.cs.cmu.edu/OpenPose/models/" POSE_FOLDER="pose/" FACE_FOLDER="face/" +HAND_FOLDER="hand/" # ------------------------- POSE MODELS ------------------------- # Body (COCO) @@ -23,3 +24,8 @@ wget -c ${OPENPOSE_URL}${MPI_MODEL} -P ${MPI_FOLDER} # Face FACE_MODEL=${FACE_FOLDER}"pose_iter_116000.caffemodel" wget -c ${OPENPOSE_URL}${FACE_MODEL} -P ${FACE_FOLDER} + +# "------------------------- HAND MODELS -------------------------" +# Hand +HAND_MODEL=$HAND_FOLDER"pose_iter_102000.caffemodel" +wget -c ${OPENPOSE_URL}${HAND_MODEL} -P ${HAND_FOLDER} diff --git a/models/hand/pose_deploy.prototxt b/models/hand/pose_deploy.prototxt new file mode 100755 index 00000000..3554c3a0 --- /dev/null +++ b/models/hand/pose_deploy.prototxt @@ -0,0 +1,1756 @@ +input: "image" +input_dim: 1 # Original: 2 +input_dim: 3 # It crashes if not left to 3 +input_dim: 1 # Original: 368 +input_dim: 1 # Original: 368 +layer { + name: "conv1_1" + type: "Convolution" + bottom: "image" + top: "conv1_1" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 64 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu1_1" + type: "ReLU" + bottom: "conv1_1" + top: "conv1_1" +} +layer { + name: "conv1_2" + type: "Convolution" + bottom: "conv1_1" + top: "conv1_2" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 64 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu1_2" + type: "ReLU" + bottom: "conv1_2" + top: "conv1_2" +} +layer { + name: "pool1_stage1" + type: "Pooling" + bottom: "conv1_2" + top: "pool1_stage1" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "conv2_1" + type: "Convolution" + bottom: "pool1_stage1" + top: "conv2_1" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu2_1" + type: "ReLU" + bottom: "conv2_1" + top: "conv2_1" +} +layer { + name: "conv2_2" + type: "Convolution" + bottom: "conv2_1" + top: "conv2_2" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu2_2" + type: "ReLU" + bottom: "conv2_2" + top: "conv2_2" +} +layer { + name: "pool2_stage1" + type: "Pooling" + bottom: "conv2_2" + top: "pool2_stage1" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "conv3_1" + type: "Convolution" + bottom: "pool2_stage1" + top: "conv3_1" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 256 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu3_1" + type: "ReLU" + bottom: "conv3_1" + top: "conv3_1" +} +layer { + name: "conv3_2" + type: "Convolution" + bottom: "conv3_1" + top: "conv3_2" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 256 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu3_2" + type: "ReLU" + bottom: "conv3_2" + top: "conv3_2" +} +layer { + name: "conv3_3" + type: "Convolution" + bottom: "conv3_2" + top: "conv3_3" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 256 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu3_3" + type: "ReLU" + bottom: "conv3_3" + top: "conv3_3" +} +layer { + name: "conv3_4" + type: "Convolution" + bottom: "conv3_3" + top: "conv3_4" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 256 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu3_4" + type: "ReLU" + bottom: "conv3_4" + top: "conv3_4" +} +layer { + name: "pool3_stage1" + type: "Pooling" + bottom: "conv3_4" + top: "pool3_stage1" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "conv4_1" + type: "Convolution" + bottom: "pool3_stage1" + top: "conv4_1" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 512 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu4_1" + type: "ReLU" + bottom: "conv4_1" + top: "conv4_1" +} +layer { + name: "conv4_2" + type: "Convolution" + bottom: "conv4_1" + top: "conv4_2" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 512 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu4_2" + type: "ReLU" + bottom: "conv4_2" + top: "conv4_2" +} +layer { + name: "conv4_3" + type: "Convolution" + bottom: "conv4_2" + top: "conv4_3" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 512 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu4_3" + type: "ReLU" + bottom: "conv4_3" + top: "conv4_3" +} +layer { + name: "conv4_4" + type: "Convolution" + bottom: "conv4_3" + top: "conv4_4" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 512 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu4_4" + type: "ReLU" + bottom: "conv4_4" + top: "conv4_4" +} +layer { + name: "conv5_1" + type: "Convolution" + bottom: "conv4_4" + top: "conv5_1" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 512 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu5_1" + type: "ReLU" + bottom: "conv5_1" + top: "conv5_1" +} +layer { + name: "conv5_2" + type: "Convolution" + bottom: "conv5_1" + top: "conv5_2" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 512 + pad: 1 + kernel_size: 3 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu5_2" + type: "ReLU" + bottom: "conv5_2" + top: "conv5_2" +} +layer { + name: "conv5_3_CPM" + type: "Convolution" + bottom: "conv5_2" + top: "conv5_3_CPM" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 1 + kernel_size: 3 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu5_4_stage1_3" + type: "ReLU" + bottom: "conv5_3_CPM" + top: "conv5_3_CPM" +} +layer { + name: "conv6_1_CPM" + type: "Convolution" + bottom: "conv5_3_CPM" + top: "conv6_1_CPM" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 512 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "relu6_4_stage1_1" + type: "ReLU" + bottom: "conv6_1_CPM" + top: "conv6_1_CPM" +} +layer { + name: "conv6_2_CPM" + type: "Convolution" + bottom: "conv6_1_CPM" + top: "conv6_2_CPM" + param { + lr_mult: 1.0 + decay_mult: 1 + } + param { + lr_mult: 2.0 + decay_mult: 0 + } + convolution_param { + num_output: 22 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "concat_stage2" + type: "Concat" + bottom: "conv6_2_CPM" + bottom: "conv5_3_CPM" + top: "concat_stage2" + concat_param { + axis: 1 + } +} +layer { + name: "Mconv1_stage2" + type: "Convolution" + bottom: "concat_stage2" + top: "Mconv1_stage2" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_2_stage2_1" + type: "ReLU" + bottom: "Mconv1_stage2" + top: "Mconv1_stage2" +} +layer { + name: "Mconv2_stage2" + type: "Convolution" + bottom: "Mconv1_stage2" + top: "Mconv2_stage2" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_3_stage2_2" + type: "ReLU" + bottom: "Mconv2_stage2" + top: "Mconv2_stage2" +} +layer { + name: "Mconv3_stage2" + type: "Convolution" + bottom: "Mconv2_stage2" + top: "Mconv3_stage2" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_4_stage2_3" + type: "ReLU" + bottom: "Mconv3_stage2" + top: "Mconv3_stage2" +} +layer { + name: "Mconv4_stage2" + type: "Convolution" + bottom: "Mconv3_stage2" + top: "Mconv4_stage2" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_5_stage2_4" + type: "ReLU" + bottom: "Mconv4_stage2" + top: "Mconv4_stage2" +} +layer { + name: "Mconv5_stage2" + type: "Convolution" + bottom: "Mconv4_stage2" + top: "Mconv5_stage2" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_6_stage2_5" + type: "ReLU" + bottom: "Mconv5_stage2" + top: "Mconv5_stage2" +} +layer { + name: "Mconv6_stage2" + type: "Convolution" + bottom: "Mconv5_stage2" + top: "Mconv6_stage2" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_7_stage2_6" + type: "ReLU" + bottom: "Mconv6_stage2" + top: "Mconv6_stage2" +} +layer { + name: "Mconv7_stage2" + type: "Convolution" + bottom: "Mconv6_stage2" + top: "Mconv7_stage2" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 22 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "concat_stage3" + type: "Concat" + bottom: "Mconv7_stage2" + bottom: "conv5_3_CPM" + top: "concat_stage3" + concat_param { + axis: 1 + } +} +layer { + name: "Mconv1_stage3" + type: "Convolution" + bottom: "concat_stage3" + top: "Mconv1_stage3" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_2_stage3_1" + type: "ReLU" + bottom: "Mconv1_stage3" + top: "Mconv1_stage3" +} +layer { + name: "Mconv2_stage3" + type: "Convolution" + bottom: "Mconv1_stage3" + top: "Mconv2_stage3" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_3_stage3_2" + type: "ReLU" + bottom: "Mconv2_stage3" + top: "Mconv2_stage3" +} +layer { + name: "Mconv3_stage3" + type: "Convolution" + bottom: "Mconv2_stage3" + top: "Mconv3_stage3" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_4_stage3_3" + type: "ReLU" + bottom: "Mconv3_stage3" + top: "Mconv3_stage3" +} +layer { + name: "Mconv4_stage3" + type: "Convolution" + bottom: "Mconv3_stage3" + top: "Mconv4_stage3" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_5_stage3_4" + type: "ReLU" + bottom: "Mconv4_stage3" + top: "Mconv4_stage3" +} +layer { + name: "Mconv5_stage3" + type: "Convolution" + bottom: "Mconv4_stage3" + top: "Mconv5_stage3" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_6_stage3_5" + type: "ReLU" + bottom: "Mconv5_stage3" + top: "Mconv5_stage3" +} +layer { + name: "Mconv6_stage3" + type: "Convolution" + bottom: "Mconv5_stage3" + top: "Mconv6_stage3" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_7_stage3_6" + type: "ReLU" + bottom: "Mconv6_stage3" + top: "Mconv6_stage3" +} +layer { + name: "Mconv7_stage3" + type: "Convolution" + bottom: "Mconv6_stage3" + top: "Mconv7_stage3" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 22 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "concat_stage4" + type: "Concat" + bottom: "Mconv7_stage3" + bottom: "conv5_3_CPM" + top: "concat_stage4" + concat_param { + axis: 1 + } +} +layer { + name: "Mconv1_stage4" + type: "Convolution" + bottom: "concat_stage4" + top: "Mconv1_stage4" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_2_stage4_1" + type: "ReLU" + bottom: "Mconv1_stage4" + top: "Mconv1_stage4" +} +layer { + name: "Mconv2_stage4" + type: "Convolution" + bottom: "Mconv1_stage4" + top: "Mconv2_stage4" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_3_stage4_2" + type: "ReLU" + bottom: "Mconv2_stage4" + top: "Mconv2_stage4" +} +layer { + name: "Mconv3_stage4" + type: "Convolution" + bottom: "Mconv2_stage4" + top: "Mconv3_stage4" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_4_stage4_3" + type: "ReLU" + bottom: "Mconv3_stage4" + top: "Mconv3_stage4" +} +layer { + name: "Mconv4_stage4" + type: "Convolution" + bottom: "Mconv3_stage4" + top: "Mconv4_stage4" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_5_stage4_4" + type: "ReLU" + bottom: "Mconv4_stage4" + top: "Mconv4_stage4" +} +layer { + name: "Mconv5_stage4" + type: "Convolution" + bottom: "Mconv4_stage4" + top: "Mconv5_stage4" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_6_stage4_5" + type: "ReLU" + bottom: "Mconv5_stage4" + top: "Mconv5_stage4" +} +layer { + name: "Mconv6_stage4" + type: "Convolution" + bottom: "Mconv5_stage4" + top: "Mconv6_stage4" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_7_stage4_6" + type: "ReLU" + bottom: "Mconv6_stage4" + top: "Mconv6_stage4" +} +layer { + name: "Mconv7_stage4" + type: "Convolution" + bottom: "Mconv6_stage4" + top: "Mconv7_stage4" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 22 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "concat_stage5" + type: "Concat" + bottom: "Mconv7_stage4" + bottom: "conv5_3_CPM" + top: "concat_stage5" + concat_param { + axis: 1 + } +} +layer { + name: "Mconv1_stage5" + type: "Convolution" + bottom: "concat_stage5" + top: "Mconv1_stage5" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_2_stage5_1" + type: "ReLU" + bottom: "Mconv1_stage5" + top: "Mconv1_stage5" +} +layer { + name: "Mconv2_stage5" + type: "Convolution" + bottom: "Mconv1_stage5" + top: "Mconv2_stage5" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_3_stage5_2" + type: "ReLU" + bottom: "Mconv2_stage5" + top: "Mconv2_stage5" +} +layer { + name: "Mconv3_stage5" + type: "Convolution" + bottom: "Mconv2_stage5" + top: "Mconv3_stage5" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_4_stage5_3" + type: "ReLU" + bottom: "Mconv3_stage5" + top: "Mconv3_stage5" +} +layer { + name: "Mconv4_stage5" + type: "Convolution" + bottom: "Mconv3_stage5" + top: "Mconv4_stage5" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_5_stage5_4" + type: "ReLU" + bottom: "Mconv4_stage5" + top: "Mconv4_stage5" +} +layer { + name: "Mconv5_stage5" + type: "Convolution" + bottom: "Mconv4_stage5" + top: "Mconv5_stage5" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_6_stage5_5" + type: "ReLU" + bottom: "Mconv5_stage5" + top: "Mconv5_stage5" +} +layer { + name: "Mconv6_stage5" + type: "Convolution" + bottom: "Mconv5_stage5" + top: "Mconv6_stage5" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_7_stage5_6" + type: "ReLU" + bottom: "Mconv6_stage5" + top: "Mconv6_stage5" +} +layer { + name: "Mconv7_stage5" + type: "Convolution" + bottom: "Mconv6_stage5" + top: "Mconv7_stage5" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 22 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "concat_stage6" + type: "Concat" + bottom: "Mconv7_stage5" + bottom: "conv5_3_CPM" + top: "concat_stage6" + concat_param { + axis: 1 + } +} +layer { + name: "Mconv1_stage6" + type: "Convolution" + bottom: "concat_stage6" + top: "Mconv1_stage6" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_2_stage6_1" + type: "ReLU" + bottom: "Mconv1_stage6" + top: "Mconv1_stage6" +} +layer { + name: "Mconv2_stage6" + type: "Convolution" + bottom: "Mconv1_stage6" + top: "Mconv2_stage6" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_3_stage6_2" + type: "ReLU" + bottom: "Mconv2_stage6" + top: "Mconv2_stage6" +} +layer { + name: "Mconv3_stage6" + type: "Convolution" + bottom: "Mconv2_stage6" + top: "Mconv3_stage6" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_4_stage6_3" + type: "ReLU" + bottom: "Mconv3_stage6" + top: "Mconv3_stage6" +} +layer { + name: "Mconv4_stage6" + type: "Convolution" + bottom: "Mconv3_stage6" + top: "Mconv4_stage6" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_5_stage6_4" + type: "ReLU" + bottom: "Mconv4_stage6" + top: "Mconv4_stage6" +} +layer { + name: "Mconv5_stage6" + type: "Convolution" + bottom: "Mconv4_stage6" + top: "Mconv5_stage6" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 3 + kernel_size: 7 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_6_stage6_5" + type: "ReLU" + bottom: "Mconv5_stage6" + top: "Mconv5_stage6" +} +layer { + name: "Mconv6_stage6" + type: "Convolution" + bottom: "Mconv5_stage6" + top: "Mconv6_stage6" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 128 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} +layer { + name: "Mrelu1_7_stage6_6" + type: "ReLU" + bottom: "Mconv6_stage6" + top: "Mconv6_stage6" +} +layer { + name: "Mconv7_stage6" + type: "Convolution" + bottom: "Mconv6_stage6" +# top: "Mconv7_stage6" + top: "net_output" + param { + lr_mult: 4.0 + decay_mult: 1 + } + param { + lr_mult: 8.0 + decay_mult: 0 + } + convolution_param { + num_output: 22 + pad: 0 + kernel_size: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + bias_filler { + type: "constant" + } + dilation: 1 + } +} + diff --git a/src/openpose/core/cvMatToOpInput.cpp b/src/openpose/core/cvMatToOpInput.cpp index 8bf9ba5c..6fa93e8d 100644 --- a/src/openpose/core/cvMatToOpInput.cpp +++ b/src/openpose/core/cvMatToOpInput.cpp @@ -38,7 +38,7 @@ namespace op { const auto currentScale = 1.f - i*mScaleGap; if (currentScale < 0.f || 1.f < currentScale) - error("All scales must be in the range [0, 1], i.e. 0 <= 1-num_scales*scale_gap <= 1", __LINE__, __FUNCTION__, __FILE__); + error("All scales must be in the range [0, 1], i.e. 0 <= 1-scale_number*scale_gap <= 1", __LINE__, __FUNCTION__, __FILE__); const auto netInputWidth = inputNetData.getSize(3); const auto targetWidth = fastTruncate(intRound(netInputWidth * currentScale) / 16 * 16, 1, netInputWidth); diff --git a/src/openpose/core/resizeAndMergeBase.cu b/src/openpose/core/resizeAndMergeBase.cu index 3e57c958..7927d7cb 100644 --- a/src/openpose/core/resizeAndMergeBase.cu +++ b/src/openpose/core/resizeAndMergeBase.cu @@ -93,7 +93,7 @@ namespace op // Multi-scale merging else { - // If num_scales > 1 --> scaleRatios must be set + // If scale_number > 1 --> scaleRatios must be set if (scaleRatios.size() != num) error("The scale ratios size must be equal than the number of scales.", __LINE__, __FUNCTION__, __FILE__); const auto maxScales = 10; diff --git a/src/openpose/face/faceRenderer.cpp b/src/openpose/face/faceRenderer.cpp index b1d9d8ca..9443243d 100644 --- a/src/openpose/face/faceRenderer.cpp +++ b/src/openpose/face/faceRenderer.cpp @@ -9,8 +9,10 @@ namespace op { - FaceRenderer::FaceRenderer(const Point& frameSize, const float alphaKeypoint, const float alphaHeatMap, const RenderMode renderMode) : + FaceRenderer::FaceRenderer(const Point& frameSize, const float renderThreshold, const float alphaKeypoint, + const float alphaHeatMap, const RenderMode renderMode) : Renderer{(unsigned long long)(frameSize.area() * 3), alphaKeypoint, alphaHeatMap}, + mRenderThreshold{renderThreshold}, mFrameSize{frameSize}, mRenderMode{renderMode} { @@ -75,7 +77,7 @@ namespace op { try { - renderFaceKeypointsCpu(outputData, faceKeypoints); + renderFaceKeypointsCpu(outputData, faceKeypoints, mRenderThreshold); } catch (const std::exception& e) { @@ -95,9 +97,11 @@ namespace op { cpuToGpuMemoryIfNotCopiedYet(outputData.getPtr()); // Draw faceKeypoints - cudaMemcpy(pGpuFace, faceKeypoints.getConstPtr(), faceKeypoints.getSize(0) * FACE_NUMBER_PARTS * 3 * sizeof(float), + cudaMemcpy(pGpuFace, faceKeypoints.getConstPtr(), + faceKeypoints.getSize(0) * FACE_NUMBER_PARTS * 3 * sizeof(float), cudaMemcpyHostToDevice); - renderFaceKeypointsGpu(*spGpuMemoryPtr, mFrameSize, pGpuFace, faceKeypoints.getSize(0), getAlphaKeypoint()); + renderFaceKeypointsGpu(*spGpuMemoryPtr, mFrameSize, pGpuFace, faceKeypoints.getSize(0), + mRenderThreshold, getAlphaKeypoint()); // CUDA check cudaCheck(__LINE__, __FUNCTION__, __FILE__); } diff --git a/src/openpose/face/renderFace.cpp b/src/openpose/face/renderFace.cpp index 97989c63..2f0d7853 100644 --- a/src/openpose/face/renderFace.cpp +++ b/src/openpose/face/renderFace.cpp @@ -8,7 +8,7 @@ namespace op { const std::vector COLORS{FACE_COLORS_RENDER}; - void renderFaceKeypointsCpu(Array& frameArray, const Array& faceKeypoints) + void renderFaceKeypointsCpu(Array& frameArray, const Array& faceKeypoints, const float renderThreshold) { try { @@ -20,7 +20,7 @@ namespace op const auto& pairs = FACE_PAIRS_RENDER; // Render keypoints - renderKeypointsCpu(frameArray, faceKeypoints, pairs, COLORS, thicknessCircleRatio, thicknessLineRatioWRTCircle, FACE_RENDER_THRESHOLD); + renderKeypointsCpu(frameArray, faceKeypoints, pairs, COLORS, thicknessCircleRatio, thicknessLineRatioWRTCircle, renderThreshold); } } catch (const std::exception& e) diff --git a/src/openpose/face/renderFace.cu b/src/openpose/face/renderFace.cu index ff104833..53de5cab 100644 --- a/src/openpose/face/renderFace.cu +++ b/src/openpose/face/renderFace.cu @@ -11,10 +11,9 @@ namespace op __constant__ const unsigned int PART_PAIRS_GPU[] = FACE_PAIRS_RENDER_GPU; __constant__ const float COLORS[] = {FACE_COLORS_RENDER}; - - - __global__ void renderFaceParts(float* targetPtr, const int targetWidth, const int targetHeight, const float* const facePtr, - const int numberFaces, const float threshold, const float alphaColorToAdd) + __global__ void renderFaceParts(float* targetPtr, const int targetWidth, const int targetHeight, + const float* const facePtr, const int numberPeople, const float threshold, + const float alphaColorToAdd) { const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; @@ -33,20 +32,21 @@ namespace op // Render key points renderKeypoints(targetPtr, sharedMaxs, sharedMins, sharedScaleF, - globalIdx, x, y, targetWidth, targetHeight, facePtr, PART_PAIRS_GPU, numberFaces, + globalIdx, x, y, targetWidth, targetHeight, facePtr, PART_PAIRS_GPU, numberPeople, FACE_NUMBER_PARTS, numberPartPairs, COLORS, numberColors, radius, stickwidth, threshold, alphaColorToAdd); } - void renderFaceKeypointsGpu(float* framePtr, const Point& frameSize, const float* const facePtr, const int numberFaces, - const float alphaColorToAdd) + void renderFaceKeypointsGpu(float* framePtr, const Point& frameSize, const float* const facePtr, + const int numberPeople, const float renderThreshold, const float alphaColorToAdd) { try { - if (numberFaces > 0) + if (numberPeople > 0) { const auto numBlocks = getNumberCudaBlocks(frameSize, THREADS_PER_BLOCK); - renderFaceParts<<>>(framePtr, frameSize.x, frameSize.y, facePtr, numberFaces, FACE_RENDER_THRESHOLD, + renderFaceParts<<>>(framePtr, frameSize.x, frameSize.y, facePtr, + numberPeople, renderThreshold, alphaColorToAdd); cudaCheck(__LINE__, __FUNCTION__, __FILE__); } diff --git a/src/openpose/hand/handExtractor.cpp b/src/openpose/hand/handExtractor.cpp index 80d5b6e9..6d188da6 100644 --- a/src/openpose/hand/handExtractor.cpp +++ b/src/openpose/hand/handExtractor.cpp @@ -12,9 +12,91 @@ namespace op { - HandExtractor::HandExtractor(const Point& netInputSize, const Point& netOutputSize, const std::string& modelFolder, const int gpuId, - const bool iterativeDetection) : - mIterativeDetection{iterativeDetection}, + void cropFrame(Array& handImageCrop, cv::Mat& affineMatrix, const cv::Mat& cvInputData, const Rectangle& handRectangle, + const int netInputSide, const Point& netOutputSize, const bool mirrorImage) + { + try + { + // Resize image to hands positions + const auto scaleLeftHand = handRectangle.width / (float)netInputSide; + affineMatrix = cv::Mat::eye(2,3,CV_64F); + if (mirrorImage) + affineMatrix.at(0,0) = -scaleLeftHand; + else + affineMatrix.at(0,0) = scaleLeftHand; + affineMatrix.at(1,1) = scaleLeftHand; + if (mirrorImage) + affineMatrix.at(0,2) = handRectangle.x + handRectangle.width; + else + affineMatrix.at(0,2) = handRectangle.x; + affineMatrix.at(1,2) = handRectangle.y; + cv::Mat handImage; + cv::warpAffine(cvInputData, handImage, affineMatrix, cv::Size{netOutputSize.x, netOutputSize.y}, + CV_INTER_LINEAR | CV_WARP_INVERSE_MAP, cv::BORDER_CONSTANT, cv::Scalar{0,0,0}); + // CV_INTER_CUBIC | CV_WARP_INVERSE_MAP, cv::BORDER_CONSTANT, cv::Scalar{0,0,0}); + // cv::Mat -> float* + uCharCvMatToFloatPtr(handImageCrop.getPtr(), handImage, true); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void connectKeypoints(Array& handCurrent, const float scaleInputToOutput, const int person, const cv::Mat& affineMatrix, + const float* handPeaks) + { + try + { + // Estimate keypoint locations + for (auto part = 0 ; part < handCurrent.getSize(1) ; part++) + { + const auto xyIndex = part * handCurrent.getSize(2); + const auto x = handPeaks[xyIndex]; + const auto y = handPeaks[xyIndex + 1]; + const auto score = handPeaks[xyIndex + 2]; + const auto baseIndex = handCurrent.getSize(2) * (part + person * handCurrent.getSize(1)); + handCurrent[baseIndex] = (float)(scaleInputToOutput * (affineMatrix.at(0,0)*x + affineMatrix.at(0,1)*y + + affineMatrix.at(0,2))); + handCurrent[baseIndex+1] = (float)(scaleInputToOutput * (affineMatrix.at(1,0)*x + affineMatrix.at(1,1)*y + + affineMatrix.at(1,2))); + handCurrent[baseIndex+2] = score; + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + Rectangle getHandRectangle(Array& handCurrent, const int person, const float increaseRatio, + const int handNumberParts, const float thresholdRectangle, + const Rectangle& previousHandRectangle = Rectangle{}) + { + try + { + // Initial Rectangle + auto handRectangle = getKeypointsRectangle(handCurrent, person, handNumberParts, thresholdRectangle); + // Get final width + auto finalWidth = fastMax(handRectangle.width, handRectangle.height) * increaseRatio; + if (previousHandRectangle.width > 0 && previousHandRectangle.height > 0) + finalWidth = fastMax(handRectangle.width, 0.85f + * fastMax(previousHandRectangle.width, previousHandRectangle.height)); + // Update Rectangle + handRectangle.recenter(finalWidth, finalWidth); + return handRectangle; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return Rectangle{}; + } + } + + HandExtractor::HandExtractor(const Point& netInputSize, const Point& netOutputSize, + const std::string& modelFolder, const int gpuId, const unsigned short numberScales, + const float rangeScales) : + mMultiScaleNumberAndRange{std::make_pair(numberScales, rangeScales)}, mNetOutputSize{netOutputSize}, spNet{std::make_shared(std::array{1, 3, mNetOutputSize.y, mNetOutputSize.x}, modelFolder + HAND_PROTOTXT, modelFolder + HAND_TRAINED_MODEL, gpuId)}, @@ -24,7 +106,6 @@ namespace op { try { -error("Hands extraction is not implemented yet. COMING SOON!", __LINE__, __FUNCTION__, __FILE__); checkE(netOutputSize.x, netInputSize.x, "Net input and output size must be equal.", __LINE__, __FUNCTION__, __FILE__); checkE(netOutputSize.y, netInputSize.y, "Net input and output size must be equal.", __LINE__, __FUNCTION__, __FILE__); checkE(netInputSize.x, netInputSize.y, "Net input size must be squared.", __LINE__, __FUNCTION__, __FILE__); @@ -73,10 +154,103 @@ error("Hands extraction is not implemented yet. COMING SOON!", __LINE__, __FUNCT { try { -error("Hands extraction is not implemented yet. COMING SOON!", __LINE__, __FUNCTION__, __FILE__); - UNUSED(handRectangles); - UNUSED(cvInputData); - UNUSED(scaleInputToOutput); + if (!handRectangles.empty()) + { + // Security checks + if (cvInputData.empty()) + error("Empty cvInputData.", __LINE__, __FUNCTION__, __FILE__); + + // Fix parameters + const auto netInputSide = fastMin(mNetOutputSize.x, mNetOutputSize.y); + + // Set hand size + const auto numberPeople = (int)handRectangles.size(); + mHandKeypoints[0].reset({numberPeople, HAND_NUMBER_PARTS, 3}, 0); + mHandKeypoints[1].reset(mHandKeypoints[0].getSize(), 0); + + // // Debugging + // cv::Mat cvInputDataCopied = cvInputData.clone(); + // Extract hand keypoints for each person + for (auto hand = 0 ; hand < 2 ; hand++) + { + // Parameters + auto& handCurrent = mHandKeypoints[hand]; + const bool mirrorImage = (hand == 0); + for (auto person = 0 ; person < numberPeople ; person++) + { + const auto& handRectangle = handRectangles.at(person).at(hand); + // Only consider faces with a minimum pixel area + const auto minHandSize = fastMin(handRectangle.width, handRectangle.height); + // // Debugging -> red rectangle + // if (handRectangle.width > 0) + // cv::rectangle(cvInputDataCopied, + // cv::Point{intRound(handRectangle.x), intRound(handRectangle.y)}, + // cv::Point{intRound(handRectangle.x + handRectangle.width), + // intRound(handRectangle.y + handRectangle.height)}, + // cv::Scalar{(hand * 255.f),0.f,255.f}, 2); + // Get parts + if (minHandSize > 1 && handRectangle.area() > 10) + { + // Single-scale detection + if (mMultiScaleNumberAndRange.first == 1) + { + // // Debugging -> green rectangle overwriting red one + // if (handRectangle.width > 0) + // cv::rectangle(cvInputDataCopied, + // cv::Point{intRound(handRectangle.x), intRound(handRectangle.y)}, + // cv::Point{intRound(handRectangle.x + handRectangle.width), + // intRound(handRectangle.y + handRectangle.height)}, + // cv::Scalar{(hand * 255.f),255.f,0.f}, 2); + // Parameters + cv::Mat affineMatrix; + // Resize image to hands positions + cv::Mat -> float* + cropFrame(mHandImageCrop, affineMatrix, cvInputData, handRectangle, netInputSide, mNetOutputSize, mirrorImage); + // Deep net + Estimate keypoint locations + detectHandKeypoints(handCurrent, scaleInputToOutput, person, affineMatrix); + } + // Multi-scale detection + else + { + const auto handPtrArea = handCurrent.getSize(1) * handCurrent.getSize(2); + auto* handCurrentPtr = handCurrent.getPtr() + person * handPtrArea; + const auto numberScales = mMultiScaleNumberAndRange.first; + const auto initScale = 1.f - mMultiScaleNumberAndRange.second / 2.f; + for (auto i = 0 ; i < numberScales ; i++) + { + // Get current scale + const auto scale = initScale + mMultiScaleNumberAndRange.second * i / (numberScales-1.f); + // Process hand + Array handEstimated({1, handCurrent.getSize(1), handCurrent.getSize(2)}, 0); + const auto handRectangleScale = recenter(handRectangle, + (float)(intRound(handRectangle.width * scale) / 2 * 2), + (float)(intRound(handRectangle.height * scale) / 2 * 2)); + // // Debugging -> blue rectangle + // cv::rectangle(cvInputDataCopied, + // cv::Point{intRound(handRectangleScale.x), intRound(handRectangleScale.y)}, + // cv::Point{intRound(handRectangleScale.x + handRectangleScale.width), + // intRound(handRectangleScale.y + handRectangleScale.height)}, + // cv::Scalar{255,0,0}, 2); + // Parameters + cv::Mat affineMatrix; + // Resize image to hands positions + cv::Mat -> float* + cropFrame(mHandImageCrop, affineMatrix, cvInputData, handRectangleScale, netInputSide, mNetOutputSize, mirrorImage); + // Deep net + Estimate keypoint locations + detectHandKeypoints(handEstimated, scaleInputToOutput, 0, affineMatrix); + if (i == 0 || getAverageScore(handEstimated, 0) > getAverageScore(handCurrent, person)) + std::copy(handEstimated.getConstPtr(), handEstimated.getConstPtr() + handPtrArea, handCurrentPtr); + } + } + } + } + } + // // Debugging + // cv::imshow("cvInputDataCopied", cvInputDataCopied); + } + else + { + mHandKeypoints[0].reset(); + mHandKeypoints[1].reset(); + } } catch (const std::exception& e) { @@ -116,11 +290,30 @@ error("Hands extraction is not implemented yet. COMING SOON!", __LINE__, __FUNCT { try { -error("Hands extraction is not implemented yet. COMING SOON!", __LINE__, __FUNCTION__, __FILE__); - UNUSED(handCurrent); - UNUSED(scaleInputToOutput); - UNUSED(person); - UNUSED(affineMatrix); + // Deep net + // 1. Caffe deep network + auto* inputDataGpuPtr = spNet->getInputDataGpuPtr(); + cudaMemcpy(inputDataGpuPtr, mHandImageCrop.getConstPtr(), mNetOutputSize.area() * 3 * sizeof(float), cudaMemcpyHostToDevice); + spNet->forwardPass(); + + // 2. Resize heat maps + merge different scales + #ifndef CPU_ONLY + spResizeAndMergeCaffe->Forward_gpu({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()}); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + #else + spResizeAndMergeCaffe->Forward_cpu({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()}); + #endif + + // 3. Get peaks by Non-Maximum Suppression + #ifndef CPU_ONLY + spMaximumCaffe->Forward_gpu({spHeatMapsBlob.get()}, {spPeaksBlob.get()}); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + #else + spMaximumCaffe->Forward_cpu({spHeatMapsBlob.get()}, {spPeaksBlob.get()}); + #endif + + // Estimate keypoint locations + connectKeypoints(handCurrent, scaleInputToOutput, person, affineMatrix, spPeaksBlob->mutable_cpu_data()); } catch (const std::exception& e) { diff --git a/src/openpose/hand/handRenderer.cpp b/src/openpose/hand/handRenderer.cpp index 60929bbe..1a106432 100644 --- a/src/openpose/hand/handRenderer.cpp +++ b/src/openpose/hand/handRenderer.cpp @@ -10,8 +10,10 @@ namespace op { - HandRenderer::HandRenderer(const Point& frameSize, const float alphaKeypoint, const float alphaHeatMap, const RenderMode renderMode) : + HandRenderer::HandRenderer(const Point& frameSize, const float renderThreshold, const float alphaKeypoint, + const float alphaHeatMap, const RenderMode renderMode) : Renderer{(unsigned long long)(frameSize.area() * 3), alphaKeypoint, alphaHeatMap}, + mRenderThreshold{renderThreshold}, mFrameSize{frameSize}, mRenderMode{renderMode} { @@ -40,7 +42,7 @@ namespace op Renderer::initializationOnThread(); // GPU memory allocation for rendering #ifndef CPU_ONLY - cudaMalloc((void**)(&pGpuHand), 2 * HAND_MAX_HANDS * HAND_NUMBER_PARTS * 3 * sizeof(float)); + cudaMalloc((void**)(&pGpuHand), HAND_MAX_HANDS * HAND_NUMBER_PARTS * 3 * sizeof(float)); #endif log("Finished initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__); } @@ -78,7 +80,7 @@ namespace op { try { - renderHandKeypointsCpu(outputData, handKeypoints); + renderHandKeypointsCpu(outputData, handKeypoints, mRenderThreshold); } catch (const std::exception& e) { @@ -103,7 +105,7 @@ namespace op const auto handVolume = numberPeople * handArea; cudaMemcpy(pGpuHand, handKeypoints[0].getConstPtr(), handVolume * sizeof(float), cudaMemcpyHostToDevice); cudaMemcpy(pGpuHand + handVolume, handKeypoints[1].getConstPtr(), handVolume * sizeof(float), cudaMemcpyHostToDevice); - renderHandKeypointsGpu(*spGpuMemoryPtr, mFrameSize, pGpuHand, 2 * numberPeople); + renderHandKeypointsGpu(*spGpuMemoryPtr, mFrameSize, pGpuHand, 2 * numberPeople, mRenderThreshold); // CUDA check cudaCheck(__LINE__, __FUNCTION__, __FILE__); } diff --git a/src/openpose/hand/renderHand.cpp b/src/openpose/hand/renderHand.cpp index 99841c14..f0571413 100644 --- a/src/openpose/hand/renderHand.cpp +++ b/src/openpose/hand/renderHand.cpp @@ -8,7 +8,8 @@ namespace op { const std::vector COLORS{HAND_COLORS_RENDER}; - void renderHandKeypointsCpu(Array& frameArray, const std::array, 2>& handKeypoints) + void renderHandKeypointsCpu(Array& frameArray, const std::array, 2>& handKeypoints, + const float renderThreshold) { try { @@ -18,9 +19,11 @@ namespace op const auto& pairs = HAND_PAIRS_RENDER; // Render keypoints if (!frameArray.empty()) - renderKeypointsCpu(frameArray, handKeypoints[0], pairs, COLORS, thicknessCircleRatio, thicknessLineRatioWRTCircle, HAND_RENDER_THRESHOLD); + renderKeypointsCpu(frameArray, handKeypoints[0], pairs, COLORS, thicknessCircleRatio, + thicknessLineRatioWRTCircle, renderThreshold); if (!frameArray.empty()) - renderKeypointsCpu(frameArray, handKeypoints[1], pairs, COLORS, thicknessCircleRatio, thicknessLineRatioWRTCircle, HAND_RENDER_THRESHOLD); + renderKeypointsCpu(frameArray, handKeypoints[1], pairs, COLORS, thicknessCircleRatio, + thicknessLineRatioWRTCircle, renderThreshold); } catch (const std::exception& e) { diff --git a/src/openpose/hand/renderHand.cu b/src/openpose/hand/renderHand.cu index da6a6516..44ddc115 100644 --- a/src/openpose/hand/renderHand.cu +++ b/src/openpose/hand/renderHand.cu @@ -38,8 +38,8 @@ namespace op radius, stickwidth, threshold, alphaColorToAdd); } - void renderHandKeypointsGpu(float* framePtr, const Point& frameSize, const float* const handsPtr, const int numberHands, - const float alphaColorToAdd) + void renderHandKeypointsGpu(float* framePtr, const Point& frameSize, const float* const handsPtr, + const int numberHands, const float renderThreshold, const float alphaColorToAdd) { try { @@ -49,7 +49,7 @@ namespace op dim3 numBlocks; std::tie(threadsPerBlock, numBlocks) = getNumberCudaThreadsAndBlocks(frameSize); renderHandsParts<<>>(framePtr, frameSize.x, frameSize.y, handsPtr, - numberHands, HAND_RENDER_THRESHOLD, alphaColorToAdd); + numberHands, renderThreshold, alphaColorToAdd); cudaCheck(__LINE__, __FUNCTION__, __FILE__); } } diff --git a/src/openpose/pose/poseRenderer.cpp b/src/openpose/pose/poseRenderer.cpp index 09953b4a..1f1bdda2 100644 --- a/src/openpose/pose/poseRenderer.cpp +++ b/src/openpose/pose/poseRenderer.cpp @@ -40,13 +40,14 @@ namespace op } PoseRenderer::PoseRenderer(const Point& heatMapsSize, const Point& outputSize, const PoseModel poseModel, - const std::shared_ptr& poseExtractor, const bool blendOriginalFrame, - const float alphaKeypoint, const float alphaHeatMap, const unsigned int elementToRender, - const RenderMode renderMode) : + const std::shared_ptr& poseExtractor, const float renderThreshold, + const bool blendOriginalFrame, const float alphaKeypoint, const float alphaHeatMap, + const unsigned int elementToRender, const RenderMode renderMode) : // #body elements to render = #body parts (size()) + #body part pair connections + 3 (+whole pose +whole heatmaps +PAFs) // POSE_BODY_PART_MAPPING crashes on Windows, replaced by getPoseBodyPartMapping Renderer{(unsigned long long)(outputSize.area() * 3), alphaKeypoint, alphaHeatMap, elementToRender, (unsigned int)(getPoseBodyPartMapping(poseModel).size() + POSE_BODY_PART_PAIRS[(int)poseModel].size()/2 + 3)}, // mNumberElementsToRender + mRenderThreshold{renderThreshold}, mHeatMapsSize{heatMapsSize}, mOutputSize{outputSize}, mPoseModel{poseModel}, @@ -178,7 +179,7 @@ namespace op // CPU rendering // Draw poseKeypoints if (elementRendered == 0) - renderPoseKeypointsCpu(outputData, poseKeypoints, mPoseModel, mBlendOriginalFrame); + renderPoseKeypointsCpu(outputData, poseKeypoints, mPoseModel, mRenderThreshold, mBlendOriginalFrame); // Draw heat maps / PAFs else { @@ -219,7 +220,7 @@ namespace op cudaMemcpy(pGpuPose, poseKeypoints.getConstPtr(), numberPeople * numberBodyParts * 3 * sizeof(float), cudaMemcpyHostToDevice); renderPoseKeypointsGpu(*spGpuMemoryPtr, mPoseModel, numberPeople, mOutputSize, pGpuPose, - mShowGooglyEyes, mBlendOriginalFrame, getAlphaKeypoint()); + mRenderThreshold, mShowGooglyEyes, mBlendOriginalFrame, getAlphaKeypoint()); } else { diff --git a/src/openpose/pose/renderPose.cpp b/src/openpose/pose/renderPose.cpp index 43a7561f..6577cc09 100644 --- a/src/openpose/pose/renderPose.cpp +++ b/src/openpose/pose/renderPose.cpp @@ -9,7 +9,8 @@ namespace op const std::vector COCO_COLORS{POSE_COCO_COLORS_RENDER}; const std::vector MPI_COLORS{POSE_MPI_COLORS_RENDER}; - void renderPoseKeypointsCpu(Array& frameArray, const Array& poseKeypoints, const PoseModel poseModel, const bool blendOriginalFrame) + void renderPoseKeypointsCpu(Array& frameArray, const Array& poseKeypoints, const PoseModel poseModel, + const float renderThreshold, const bool blendOriginalFrame) { try { @@ -29,7 +30,8 @@ namespace op const auto& colors = (poseModel == PoseModel::COCO_18 ? COCO_COLORS : MPI_COLORS); // Render keypoints - renderKeypointsCpu(frameArray, poseKeypoints, pairs, colors, thicknessCircleRatio, thicknessLineRatioWRTCircle, POSE_RENDER_THRESHOLD); + renderKeypointsCpu(frameArray, poseKeypoints, pairs, colors, thicknessCircleRatio, + thicknessLineRatioWRTCircle, renderThreshold); } } catch (const std::exception& e) diff --git a/src/openpose/pose/renderPose.cu b/src/openpose/pose/renderPose.cu index 37b4659e..852344fb 100644 --- a/src/openpose/pose/renderPose.cu +++ b/src/openpose/pose/renderPose.cu @@ -315,8 +315,9 @@ namespace op } } - void renderPoseKeypointsGpu(float* framePtr, const PoseModel poseModel, const int numberPeople, const Point& frameSize, - const float* const posePtr, const bool googlyEyes, const bool blendOriginalFrame, const float alphaBlending) + void renderPoseKeypointsGpu(float* framePtr, const PoseModel poseModel, const int numberPeople, + const Point& frameSize, const float* const posePtr, const float renderThreshold, + const bool googlyEyes, const bool blendOriginalFrame, const float alphaBlending) { try { @@ -334,13 +335,13 @@ namespace op if (poseModel == PoseModel::COCO_18) renderPoseCoco<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, - POSE_RENDER_THRESHOLD, googlyEyes, blendOriginalFrame, alphaBlending); + renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending); else if (poseModel == PoseModel::BODY_22) renderPoseBody22<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, - POSE_RENDER_THRESHOLD, googlyEyes, blendOriginalFrame, alphaBlending); + renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending); else if (poseModel == PoseModel::MPI_15 || poseModel == PoseModel::MPI_15_4) renderPoseMpi29Parts<<>>(framePtr, frameSize.x, frameSize.y, posePtr, - numberPeople, POSE_RENDER_THRESHOLD, blendOriginalFrame, alphaBlending); + numberPeople, renderThreshold, blendOriginalFrame, alphaBlending); else error("Invalid Model.", __LINE__, __FUNCTION__, __FILE__); cudaCheck(__LINE__, __FUNCTION__, __FILE__); diff --git a/src/openpose/utilities/flagsToOpenPose.cpp b/src/openpose/utilities/flagsToOpenPose.cpp index a15ca1ac..3387b059 100644 --- a/src/openpose/utilities/flagsToOpenPose.cpp +++ b/src/openpose/utilities/flagsToOpenPose.cpp @@ -137,41 +137,6 @@ namespace op } } - DetectionMode flagsToDetectionMode(const int handDetectionModeFlag, const std::shared_ptr& producer) - { - try - { - if (handDetectionModeFlag == -1) - { - if (producer == nullptr) - error("Since there is no default producer, `hand_detection_mode` must be set.", __LINE__, __FUNCTION__, __FILE__); - const auto producerType = producer->getType(); - if (producerType == ProducerType::Webcam) - return DetectionMode::Fast; - else if (producerType == ProducerType::ImageDirectory) - return DetectionMode::Iterative; - else if (producerType == ProducerType::Video) - return DetectionMode::Tracking; - } - else if (handDetectionModeFlag == 0) - return DetectionMode::Fast; - else if (handDetectionModeFlag == 1) - return DetectionMode::Iterative; - else if (handDetectionModeFlag == 2) - return DetectionMode::Tracking; - else if (handDetectionModeFlag == 3) - return DetectionMode::IterativeAndTracking; - // else - error("Undefined DetectionMode selected.", __LINE__, __FUNCTION__, __FILE__); - return DetectionMode::Fast; - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return DetectionMode::Fast; - } - } - RenderMode flagsToRenderMode(const int renderFlag, const int renderPoseFlag) { try diff --git a/src/openpose/wrapper/wrapperStructFace.cpp b/src/openpose/wrapper/wrapperStructFace.cpp index 9b960193..44100bde 100644 --- a/src/openpose/wrapper/wrapperStructFace.cpp +++ b/src/openpose/wrapper/wrapperStructFace.cpp @@ -3,12 +3,14 @@ namespace op { WrapperStructFace::WrapperStructFace(const bool enable_, const Point& netInputSize_, const RenderMode renderMode_, - const float alphaKeypoint_, const float alphaHeatMap_) : + const float alphaKeypoint_, const float alphaHeatMap_, + const float renderThreshold_) : enable{enable_}, netInputSize{netInputSize_}, renderMode{renderMode_}, alphaKeypoint{alphaKeypoint_}, - alphaHeatMap{alphaHeatMap_} + alphaHeatMap{alphaHeatMap_}, + renderThreshold{renderThreshold_} { } } diff --git a/src/openpose/wrapper/wrapperStructHand.cpp b/src/openpose/wrapper/wrapperStructHand.cpp index d7852c9d..cc44109b 100644 --- a/src/openpose/wrapper/wrapperStructHand.cpp +++ b/src/openpose/wrapper/wrapperStructHand.cpp @@ -2,14 +2,19 @@ namespace op { - WrapperStructHand::WrapperStructHand(const bool enable_, const Point& netInputSize_, const DetectionMode detectionMode_, - const RenderMode renderMode_, const float alphaKeypoint_, const float alphaHeatMap_) : + WrapperStructHand::WrapperStructHand(const bool enable_, const Point& netInputSize_, const int scalesNumber_, + const float scaleRange_, const bool tracking_, const RenderMode renderMode_, + const float alphaKeypoint_, const float alphaHeatMap_, + const float renderThreshold_) : enable{enable_}, netInputSize{netInputSize_}, - detectionMode{detectionMode_}, + scalesNumber{scalesNumber_}, + scaleRange{scaleRange_}, + tracking{tracking_}, renderMode{renderMode_}, alphaKeypoint{alphaKeypoint_}, - alphaHeatMap{alphaHeatMap_} + alphaHeatMap{alphaHeatMap_}, + renderThreshold{renderThreshold_} { } } diff --git a/src/openpose/wrapper/wrapperStructPose.cpp b/src/openpose/wrapper/wrapperStructPose.cpp index 9fc823a4..cc789576 100644 --- a/src/openpose/wrapper/wrapperStructPose.cpp +++ b/src/openpose/wrapper/wrapperStructPose.cpp @@ -10,7 +10,7 @@ namespace op const float alphaHeatMap_, const int defaultPartToRender_, const std::string& modelFolder_, const std::vector& heatMapTypes_, - const ScaleMode heatMapScale_) : + const ScaleMode heatMapScale_, const float renderThreshold_) : netInputSize{netInputSize_}, outputSize{outputSize_}, keypointScale{keypointScale_}, @@ -26,7 +26,8 @@ namespace op defaultPartToRender{defaultPartToRender_}, modelFolder{modelFolder_}, heatMapTypes{heatMapTypes_}, - heatMapScale{heatMapScale_} + heatMapScale{heatMapScale_}, + renderThreshold{renderThreshold_} { } }