diff --git a/.gitignore b/.gitignore index cad663f2..93ca4845 100644 --- a/.gitignore +++ b/.gitignore @@ -118,6 +118,4 @@ html/ *.mov ######################### Validation Scripts & Testing ######################### -*validation.sh -validation*.sh output*/ diff --git a/README.md b/README.md index 5baa2bb8..a910f053 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ OpenPose is a **library for real-time multi-person keypoint detection and multi- OpenPose represents the **first real-time system to jointly detect human body, hand and facial keypoints (in total 130 keypoints) on single images**. In addition, the system computational performance on body keypoint estimation is invariant to the number of detected people in the image. -OpenPose is freely available for free non-commercial use, and may be redistributed under these conditions. Please, see the [license](LICENSE) for further details. Contact us for commercial purposes. +OpenPose is freely available for free non-commercial use, and may be redistributed under these conditions. Please, see the [license](LICENSE) for further details. Contact [Yaser Sheikh](http://www.cs.cmu.edu/~yaser/) for commercial purposes. @@ -77,7 +77,7 @@ The pose estimation work is based on the C++ code from [the ECCV 2016 demo](http 2. [OpenPose Wrapper](#openpose-wrapper) 3. [OpenPose Library](#openpose-library) 4. [Output](#output) -5. [OpenPose Benchmark](#openpose-benchmark) +5. [Speed Up Openpose And Benchmark](#speed-up-openpose-and-benchmark) 6. [Send Us Your Feedback!](#send-us-your-feedback) 7. [Citation](#citation) 8. [Other Contributors](#other-contributors) @@ -191,6 +191,4 @@ Please cite the papers in your publications if it helps your research: ## Other Contributors -We would like to thank the following people who also contributed to OpenPose: - -1. [Helen Medina](https://github.com/helen-medina): For moving OpenPose to Windows (Visual Studio), making it work there and creating the Windows branch. +We would like to thank all the people who helped OpenPose in any way. The main contributors are listed in [doc/contributors.md](doc/contributors.md). diff --git a/doc/release_notes.md b/doc/release_notes.md index 45fdab29..ea88adae 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -44,18 +44,30 @@ OpenPose Library - Release Notes -## Current version (future OpenPose 1.0.0rc4) +## OpenPose 1.0.0 1. Main improvements: - 1. Increased accuracy on multi-scale (added `Datum::scaleRatios` to save the relative scale ratio when multi-scale). - 2. Increased speed ~3-5% by adding CPU rendering and setting it as default rendering. - 3. Check() functions give more feedback. - 4. WCocoJsonSaver finished and removed its 3599-image limit. - 5. Added `camera_fps` so generated video will use that frame rate. - 6. Improved documentation. + 1. Hands and face now use `Maximum` instead of `Nms`, since there is only 1 person / detection. + 2. Increased accuracy on multi-scale (added `Datum::scaleRatios` to save the relative scale ratio when multi-scale). + 3. Increased speed ~5% by adding CPU rendering (but GPU is the default rendering). + 4. Rendering colors modified, visually better results. + 5. Check() functions give more feedback. + 6. WCocoJsonSaver finished and removed its 3599-image limit. + 7. Added `camera_fps` so generated video will use that frame rate. + 8. Reduced the number of printed information messages. Default logging priority threshold increased to Priority::Max. + 9. Google flags to OpenPose configuration parameters reader moved from each demo to utilities/flagsToOpenPose. + 10. Nms classes do not use `numberParts` for `Reshape`, they deduce the value. + 11. Improved documentation. 2. Functions or parameters renamed: 1. Render flags renamed in the demo in order to incorporate the CPU/GPU rendering. + 2. Keypoints saved in JSON files (`write_keypoint_json`) are now saved as `pose_keypoints`, `face_keypoints`, `hand_left_keypoints`, and `hand_right_keypoints`. They all were previously saved as `body_parts`. 3. Main bugs fixed: 1. Fixed bug in Array::getConstCvMat() if mVolume=0, now returning empty cv::Mat. 2. Fixed bug: `--process_real_time` threw error with webcam. 3. Fixed bug: Face not working when input and output resolutions are different. 4. Fixed some bugs that prevented debug version to run. + 5. Face saved in JSON files were called `body_parts`. Now they are called `face_keypoints`. + + + +## Current version (future OpenPose 1.0.1) +1. No changes yet. diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index 805ddf26..14a87e9c 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -16,7 +16,6 @@ // This file should only be used for the user to take specific examples. // C++ std library dependencies -#include #include // `std::chrono::` functions and classes, e.g. std::chrono::milliseconds #include #include // std::this_thread @@ -187,10 +186,12 @@ int openPoseDemo() op::log("Configuring OpenPose wrapper.", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__); op::Wrapper> opWrapper; // Pose configuration (use WrapperStructPose{} for default and recommended configuration) - const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, FLAGS_num_gpu_start, - FLAGS_num_scales, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, - !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, - FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, op::ScaleMode::UnsignedChar}; + const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, + FLAGS_num_gpu_start, FLAGS_num_scales, (float)FLAGS_scale_gap, + op::flagsToRenderMode(FLAGS_render_pose), poseModel, + !FLAGS_disable_blending, (float)FLAGS_alpha_pose, + (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, + heatMapTypes, op::ScaleMode::UnsignedChar}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_render_face, FLAGS_render_pose), (float)FLAGS_alpha_face, (float)FLAGS_alpha_heatmap_face}; diff --git a/examples/tutorial_wrapper/1_user_asynchronous.cpp b/examples/tutorial_wrapper/1_user_asynchronous.cpp index 64893e47..89eaea0a 100644 --- a/examples/tutorial_wrapper/1_user_asynchronous.cpp +++ b/examples/tutorial_wrapper/1_user_asynchronous.cpp @@ -259,10 +259,12 @@ int openPoseTutorialWrapper1() // Configure OpenPose op::Wrapper> opWrapper{op::ThreadManagerMode::Asynchronous}; // Pose configuration (use WrapperStructPose{} for default and recommended configuration) - const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, FLAGS_num_gpu_start, - FLAGS_num_scales, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, - !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, - FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale}; + const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, + FLAGS_num_gpu_start, FLAGS_num_scales, (float)FLAGS_scale_gap, + op::flagsToRenderMode(FLAGS_render_pose), poseModel, + !FLAGS_disable_blending, (float)FLAGS_alpha_pose, + (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, + heatMapTypes, heatMapScale}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_render_face, FLAGS_render_pose), (float)FLAGS_alpha_face, (float)FLAGS_alpha_heatmap_face}; diff --git a/examples/tutorial_wrapper/2_user_synchronous.cpp b/examples/tutorial_wrapper/2_user_synchronous.cpp index 07eb35ca..c1fd65c5 100644 --- a/examples/tutorial_wrapper/2_user_synchronous.cpp +++ b/examples/tutorial_wrapper/2_user_synchronous.cpp @@ -320,10 +320,12 @@ int openPoseTutorialWrapper2() const auto workerOutputOnNewThread = true; opWrapper.setWorkerOutput(wUserOutput, workerOutputOnNewThread); // Configure OpenPose - const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, FLAGS_num_gpu_start, - FLAGS_num_scales, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, - !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, - FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale}; + const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, + FLAGS_num_gpu_start, FLAGS_num_scales, (float)FLAGS_scale_gap, + op::flagsToRenderMode(FLAGS_render_pose), poseModel, + !FLAGS_disable_blending, (float)FLAGS_alpha_pose, + (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, + heatMapTypes, heatMapScale}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_render_face, FLAGS_render_pose), (float)FLAGS_alpha_face, (float)FLAGS_alpha_heatmap_face}; diff --git a/include/openpose/core/maximumBase.hpp b/include/openpose/core/maximumBase.hpp new file mode 100644 index 00000000..e5343f42 --- /dev/null +++ b/include/openpose/core/maximumBase.hpp @@ -0,0 +1,15 @@ +#ifndef OPENPOSE_CORE_MAXIMUM_BASE_HPP +#define OPENPOSE_CORE_MAXIMUM_BASE_HPP + +#include + +namespace op +{ + template + void maximumCpu(T* targetPtr, const T* const sourcePtr, const std::array& targetSize, const std::array& sourceSize); + + template + void maximumGpu(T* targetPtr, const T* const sourcePtr, const std::array& targetSize, const std::array& sourceSize); +} + +#endif // OPENPOSE_CORE_MAXIMUM_BASE_HPP diff --git a/include/openpose/core/maximumCaffe.hpp b/include/openpose/core/maximumCaffe.hpp new file mode 100644 index 00000000..8c7ae397 --- /dev/null +++ b/include/openpose/core/maximumCaffe.hpp @@ -0,0 +1,39 @@ +#ifdef USE_CAFFE +#ifndef OPENPOSE_CORE_MAXIMUM_CAFFE_HPP +#define OPENPOSE_CORE_MAXIMUM_CAFFE_HPP + +#include +#include "caffe/blob.hpp" + +namespace op +{ + // It mostly follows the Caffe::layer implementation, so Caffe users can easily use it. However, in order to keep the compatibility with any generic Caffe version, + // we keep this 'layer' inside our library rather than in the Caffe code. + template + class MaximumCaffe + { + public: + explicit MaximumCaffe(); + + virtual void LayerSetUp(const std::vector*>& bottom, const std::vector*>& top); + + virtual void Reshape(const std::vector*>& bottom, const std::vector*>& top); + + virtual inline const char* type() const { return "Maximum"; } + + virtual void Forward_cpu(const std::vector*>& bottom, const std::vector*>& top); + + virtual void Forward_gpu(const std::vector*>& bottom, const std::vector*>& top); + + virtual void Backward_cpu(const std::vector*>& top, const std::vector& propagate_down, const std::vector*>& bottom); + + virtual void Backward_gpu(const std::vector*>& top, const std::vector& propagate_down, const std::vector*>& bottom); + + private: + std::array mBottomSize; + std::array mTopSize; + }; +} + +#endif // OPENPOSE_CORE_MAXIMUM_CAFFE_HPP +#endif diff --git a/include/openpose/core/nmsCaffe.hpp b/include/openpose/core/nmsCaffe.hpp index cc393f20..80bebe0a 100644 --- a/include/openpose/core/nmsCaffe.hpp +++ b/include/openpose/core/nmsCaffe.hpp @@ -17,7 +17,7 @@ namespace op virtual void LayerSetUp(const std::vector*>& bottom, const std::vector*>& top); - virtual void Reshape(const std::vector*>& bottom, const std::vector*>& top, const int maxPeaks, const int numberParts); + virtual void Reshape(const std::vector*>& bottom, const std::vector*>& top, const int maxPeaks); virtual inline const char* type() const { return "Nms"; } diff --git a/include/openpose/face/enumClasses.hpp b/include/openpose/face/enumClasses.hpp deleted file mode 100644 index e5616ffb..00000000 --- a/include/openpose/face/enumClasses.hpp +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef OPENPOSE_FACE_ENUM_CLASSES_HPP -#define OPENPOSE_FACE_ENUM_CLASSES_HPP - -namespace op -{ - enum class FaceProperty : bool - { - NMSThreshold = 0, - Size, - }; -} - -#endif // OPENPOSE_FACE_ENUM_CLASSES_HPP - diff --git a/include/openpose/face/faceDetector.hpp b/include/openpose/face/faceDetector.hpp index a969d708..77debf53 100644 --- a/include/openpose/face/faceDetector.hpp +++ b/include/openpose/face/faceDetector.hpp @@ -6,7 +6,6 @@ #include #include #include -#include "enumClasses.hpp" namespace op { diff --git a/include/openpose/face/faceExtractor.hpp b/include/openpose/face/faceExtractor.hpp index 870f6be1..9830ed2d 100644 --- a/include/openpose/face/faceExtractor.hpp +++ b/include/openpose/face/faceExtractor.hpp @@ -7,11 +7,10 @@ #include #include // cv::Mat #include +#include #include -#include #include #include -#include "enumClasses.hpp" namespace op { @@ -26,19 +25,12 @@ namespace op Array getFaceKeypoints() const; - double get(const FaceProperty property) const; - - void set(const FaceProperty property, const double value); - - void increase(const FaceProperty property, const double value); - private: const Point mNetOutputSize; const Point mOutputSize; - std::array, (int)FaceProperty::Size> mProperties; std::shared_ptr spNet; std::shared_ptr> spResizeAndMergeCaffe; - std::shared_ptr> spNmsCaffe; + std::shared_ptr> spMaximumCaffe; Array mFaceImageCrop; Array mFaceKeypoints; // Init with thread diff --git a/include/openpose/face/faceParameters.hpp b/include/openpose/face/faceParameters.hpp index a124af8e..753e2f8b 100644 --- a/include/openpose/face/faceParameters.hpp +++ b/include/openpose/face/faceParameters.hpp @@ -1,7 +1,6 @@ #ifndef OPENPOSE_FACE_FACE_PARAMETERS_HPP #define OPENPOSE_FACE_FACE_PARAMETERS_HPP -#include "enumClasses.hpp" #include namespace op @@ -18,14 +17,9 @@ namespace op // Constant parameters const auto FACE_CCN_DECREASE_FACTOR = 8.f; - const auto FACE_MAX_PEAKS = 64u; const std::string FACE_PROTOTXT{"face/pose_deploy.prototxt"}; const std::string FACE_TRAINED_MODEL{"face/pose_iter_116000.caffemodel"}; - // Default Model Parameters - // They might be modified on running time - const auto FACE_DEFAULT_NMS_THRESHOLD = 0.1f; - // Rendering parameters const auto FACE_DEFAULT_ALPHA_KEYPOINT = POSE_DEFAULT_ALPHA_KEYPOINT; const auto FACE_DEFAULT_ALPHA_HEAT_MAP = POSE_DEFAULT_ALPHA_HEAT_MAP; diff --git a/include/openpose/face/headers.hpp b/include/openpose/face/headers.hpp index 0bdeba98..31085fe5 100644 --- a/include/openpose/face/headers.hpp +++ b/include/openpose/face/headers.hpp @@ -2,7 +2,6 @@ #define OPENPOSE_FACE_HEADERS_HPP // face module -#include "enumClasses.hpp" #include "faceDetector.hpp" #include "faceExtractor.hpp" #include "faceParameters.hpp" diff --git a/include/openpose/filestream/fileStream.hpp b/include/openpose/filestream/fileStream.hpp index 5e606e83..bca24cbe 100644 --- a/include/openpose/filestream/fileStream.hpp +++ b/include/openpose/filestream/fileStream.hpp @@ -1,11 +1,13 @@ #ifndef OPENPOSE_FILESTREAM_FILE_STREAM_HPP #define OPENPOSE_FILESTREAM_FILE_STREAM_HPP +#include #include #include #include // cv::Mat #include // CV_LOAD_IMAGE_ANYDEPTH, CV_IMWRITE_PNG_COMPRESSION #include +#include #include "enumClasses.hpp" namespace op @@ -31,6 +33,8 @@ namespace op void saveImage(const cv::Mat& cvMat, const std::string& fullFilePath, const std::vector& openCvCompressionParams = {CV_IMWRITE_JPEG_QUALITY, 100, CV_IMWRITE_PNG_COMPRESSION, 9}); cv::Mat loadImage(const std::string& fullFilePath, const int openCvFlags = CV_LOAD_IMAGE_ANYDEPTH); + + std::vector, 2>> loadHandDetectorTxt(const std::string& txtFilePath); } #endif // OPENPOSE_FILESTREAM_FILE_STREAM_HPP diff --git a/include/openpose/hand/enumClasses.hpp b/include/openpose/hand/enumClasses.hpp deleted file mode 100644 index 5a363540..00000000 --- a/include/openpose/hand/enumClasses.hpp +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef OPENPOSE_HAND_ENUM_CLASSES_HPP -#define OPENPOSE_HAND_ENUM_CLASSES_HPP - -namespace op -{ - enum class HandProperty : bool - { - NMSThreshold = 0, - Size, - }; -} - -#endif // OPENPOSE_HAND_ENUM_CLASSES_HPP diff --git a/include/openpose/hand/handDetector.hpp b/include/openpose/hand/handDetector.hpp index 35f28b22..353aacc8 100644 --- a/include/openpose/hand/handDetector.hpp +++ b/include/openpose/hand/handDetector.hpp @@ -9,7 +9,6 @@ #include #include #include -#include "enumClasses.hpp" namespace op { diff --git a/include/openpose/hand/handDetectorFromTxt.hpp b/include/openpose/hand/handDetectorFromTxt.hpp new file mode 100644 index 00000000..9ddb38ed --- /dev/null +++ b/include/openpose/hand/handDetectorFromTxt.hpp @@ -0,0 +1,29 @@ +#ifndef OPENPOSE_HAND_HAND_DETECTOR_FROM_TXT_HPP +#define OPENPOSE_HAND_HAND_DETECTOR_FROM_TXT_HPP + +#include +#include +#include +#include +#include +#include + +namespace op +{ + class HandDetectorFromTxt + { + public: + explicit HandDetectorFromTxt(const std::string& txtDirectoryPath); + + std::vector, 2>> detectHands(); + + private: + const std::string mTxtDirectoryPath; + const std::vector mFilePaths; + long long mFrameNameCounter; + + DELETE_COPY(HandDetectorFromTxt); + }; +} + +#endif // OPENPOSE_HAND_HAND_DETECTOR_FROM_TXT_HPP diff --git a/include/openpose/hand/handExtractor.hpp b/include/openpose/hand/handExtractor.hpp index 97afd23b..25f420ea 100644 --- a/include/openpose/hand/handExtractor.hpp +++ b/include/openpose/hand/handExtractor.hpp @@ -8,12 +8,11 @@ #include // cv::Mat #include #include +#include #include -#include #include #include #include -#include "enumClasses.hpp" namespace op { @@ -30,19 +29,12 @@ namespace op std::array, 2> getHandKeypoints() const; - double get(const HandProperty property) const; - - void set(const HandProperty property, const double value); - - void increase(const HandProperty property, const double value); - private: const bool mIterativeDetection; const Point mNetOutputSize; - std::array, (int)HandProperty::Size> mProperties; std::shared_ptr spNet; std::shared_ptr> spResizeAndMergeCaffe; - std::shared_ptr> spNmsCaffe; + std::shared_ptr> spMaximumCaffe; Array mHandImageCrop; std::array, 2> mHandKeypoints; // Init with thread @@ -53,8 +45,7 @@ namespace op void checkThread() const; - void detectHandKeypoints(Array& handCurrent, const float scaleInputToOutput, const int person, const cv::Mat& affineMatrix, - const unsigned int handPeaksOffset); + void detectHandKeypoints(Array& handCurrent, const float scaleInputToOutput, const int person, const cv::Mat& affineMatrix); DELETE_COPY(HandExtractor); }; diff --git a/include/openpose/hand/handParameters.hpp b/include/openpose/hand/handParameters.hpp index 379effb2..edc95fcf 100644 --- a/include/openpose/hand/handParameters.hpp +++ b/include/openpose/hand/handParameters.hpp @@ -1,7 +1,6 @@ #ifndef OPENPOSE_HAND_HAND_PARAMETERS_HPP #define OPENPOSE_HAND_HAND_PARAMETERS_HPP -#include "enumClasses.hpp" #include namespace op @@ -36,14 +35,9 @@ namespace op // Constant parameters const auto HAND_CCN_DECREASE_FACTOR = 8.f; - const auto HAND_MAX_PEAKS = 64u; const std::string HAND_PROTOTXT{"hand/pose_deploy.prototxt"}; const std::string HAND_TRAINED_MODEL{"hand/pose_iter_120000.caffemodel"}; - // Default Model Parameters - // They might be modified on running time - const auto HAND_DEFAULT_NMS_THRESHOLD = 0.1f; - // Rendering parameters const auto HAND_DEFAULT_ALPHA_KEYPOINT = POSE_DEFAULT_ALPHA_KEYPOINT; const auto HAND_DEFAULT_ALPHA_HEAT_MAP = POSE_DEFAULT_ALPHA_HEAT_MAP; diff --git a/include/openpose/hand/headers.hpp b/include/openpose/hand/headers.hpp index 71b4084a..257c477e 100644 --- a/include/openpose/hand/headers.hpp +++ b/include/openpose/hand/headers.hpp @@ -2,13 +2,14 @@ #define OPENPOSE_HAND_HEADERS_HPP // hand module -#include "enumClasses.hpp" #include "handDetector.hpp" +#include "handDetectorFromTxt.hpp" #include "handExtractor.hpp" #include "handParameters.hpp" #include "handRenderer.hpp" #include "renderHand.hpp" #include "wHandDetector.hpp" +#include "wHandDetectorFromTxt.hpp" #include "wHandDetectorTracking.hpp" #include "wHandDetectorUpdate.hpp" #include "wHandExtractor.hpp" diff --git a/include/openpose/hand/wHandDetector.hpp b/include/openpose/hand/wHandDetector.hpp index 463debd6..c8f9a9d0 100644 --- a/include/openpose/hand/wHandDetector.hpp +++ b/include/openpose/hand/wHandDetector.hpp @@ -3,7 +3,7 @@ #include // std::shared_ptr #include -#include "handRenderer.hpp" +#include "handDetector.hpp" namespace op { diff --git a/include/openpose/hand/wHandDetectorFromTxt.hpp b/include/openpose/hand/wHandDetectorFromTxt.hpp new file mode 100644 index 00000000..7221f63d --- /dev/null +++ b/include/openpose/hand/wHandDetectorFromTxt.hpp @@ -0,0 +1,81 @@ +#ifndef OPENPOSE_HAND_W_HAND_DETECTOR_FROM_JSON_HPP +#define OPENPOSE_HAND_W_HAND_DETECTOR_FROM_JSON_HPP + +#include // std::shared_ptr +#include +#include "handDetectorFromTxt.hpp" + +namespace op +{ + template + class WHandDetectorFromTxt : public Worker + { + public: + explicit WHandDetectorFromTxt(const std::shared_ptr& handDetectorFromTxt); + + void initializationOnThread(); + + void work(TDatums& tDatums); + + private: + std::shared_ptr spHandDetectorFromTxt; + + DELETE_COPY(WHandDetectorFromTxt); + }; +} + + + + + +// Implementation +#include +#include +#include +#include +namespace op +{ + template + WHandDetectorFromTxt::WHandDetectorFromTxt(const std::shared_ptr& handDetectorFromTxt) : + spHandDetectorFromTxt{handDetectorFromTxt} + { + } + + template + void WHandDetectorFromTxt::initializationOnThread() + { + } + + template + void WHandDetectorFromTxt::work(TDatums& tDatums) + { + try + { + if (checkNoNullNorEmpty(tDatums)) + { + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + // Detect people hand + for (auto& tDatum : *tDatums) + tDatum.handRectangles = spHandDetectorFromTxt->detectHands(); + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, Profiler::DEFAULT_X); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + } + catch (const std::exception& e) + { + this->stop(); + tDatums = nullptr; + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + COMPILE_TEMPLATE_DATUM(WHandDetectorFromTxt); +} + +#endif // OPENPOSE_HAND_W_HAND_DETECTOR_FROM_JSON_HPP diff --git a/include/openpose/utilities/fileSystem.hpp b/include/openpose/utilities/fileSystem.hpp index de5a0b04..37a033c4 100644 --- a/include/openpose/utilities/fileSystem.hpp +++ b/include/openpose/utilities/fileSystem.hpp @@ -2,6 +2,7 @@ #define OPENPOSE_UTILITIES_FILE_SYSTEM_HPP #include +#include namespace op { diff --git a/include/openpose/utilities/flagsToOpenPose.hpp b/include/openpose/utilities/flagsToOpenPose.hpp index ebcadfb3..8d0b0609 100644 --- a/include/openpose/utilities/flagsToOpenPose.hpp +++ b/include/openpose/utilities/flagsToOpenPose.hpp @@ -1,5 +1,5 @@ -#ifndef OPENPOSE_UTILITIES_G_FLAGS_TO_OPEN_POSE_HPP -#define OPENPOSE_UTILITIES_G_FLAGS_TO_OPEN_POSE_HPP +#ifndef OPENPOSE_UTILITIES_FLAGS_TO_OPEN_POSE_HPP +#define OPENPOSE_UTILITIES_FLAGS_TO_OPEN_POSE_HPP #include // std::shared_ptr #include @@ -19,9 +19,11 @@ namespace op ProducerType flagsToProducerType(const std::string& imageDirectory, const std::string& videoPath, const int webcamIndex); std::shared_ptr flagsToProducer(const std::string& imageDirectory, const std::string& videoPath, - const int webcamIndex, const std::string& webcamResolution, const double webcamFps); + const int webcamIndex, const std::string& webcamResolution = "1280x720", + const double webcamFps = 30.); - std::vector flagsToHeatMaps(const bool heatMapsAddParts, const bool heatMapsAddBkg, const bool heatMapsAddPAFs); + std::vector flagsToHeatMaps(const bool heatMapsAddParts = false, const bool heatMapsAddBkg = false, + const bool heatMapsAddPAFs = false); DetectionMode flagsToDetectionMode(const int handDetectionModeFlag, const std::shared_ptr& producer = nullptr); @@ -30,4 +32,4 @@ namespace op Point flagsToPoint(const std::string& pointString, const std::string& pointExample = "1280x720"); } -#endif // OPENPOSE_UTILITIES_G_FLAGS_TO_OPEN_POSE_HPP +#endif // OPENPOSE_UTILITIES_FLAGS_TO_OPEN_POSE_HPP diff --git a/include/openpose/utilities/string.hpp b/include/openpose/utilities/string.hpp index 2fb29e0a..262c4285 100644 --- a/include/openpose/utilities/string.hpp +++ b/include/openpose/utilities/string.hpp @@ -2,6 +2,7 @@ #define OPENPOSE_UTILITIES_STRING_HPP #include +#include namespace op { @@ -14,6 +15,8 @@ namespace op */ template std::string toFixedLengthString(const T number, const unsigned long long stringLength = 0); + + std::vector splitString(const std::string& stringToSplit, const std::string& delimiter); } #endif // OPENPOSE_UTILITIES_STRING_HPP diff --git a/include/openpose/wrapper/wrapper.hpp b/include/openpose/wrapper/wrapper.hpp index cf6179d9..704e72c7 100644 --- a/include/openpose/wrapper/wrapper.hpp +++ b/include/openpose/wrapper/wrapper.hpp @@ -24,7 +24,9 @@ namespace op * - Asynchronous input + synchronous output: call the constructor * Wrapper(ThreadManagerMode::Synchronous, nullptr, workersOutput, irrelevantBoolean, true) */ - template>>, typename TQueue = Queue>> + template>>, + typename TQueue = Queue>> class Wrapper { public: @@ -201,7 +203,7 @@ namespace op bool mUserInputWsOnNewThread; bool mUserPostProcessingWsOnNewThread; bool mUserOutputWsOnNewThread; - unsigned int mThreadId; + unsigned long long mThreadId; bool mMultiThreadEnabled; // Workers std::vector mUserInputWs; @@ -535,7 +537,7 @@ namespace op wrapperStructInput.producerSharedPtr->set(ProducerProperty::AutoRepeat, wrapperStructInput.framesRepeat); // 2. Set finalOutputSize producerSize = Point{(int)wrapperStructInput.producerSharedPtr->get(CV_CAP_PROP_FRAME_WIDTH), - (int)wrapperStructInput.producerSharedPtr->get(CV_CAP_PROP_FRAME_HEIGHT)}; + (int)wrapperStructInput.producerSharedPtr->get(CV_CAP_PROP_FRAME_HEIGHT)}; if (wrapperStructPose.outputSize.x == -1 || wrapperStructPose.outputSize.y == -1) { if (producerSize.area() > 0) diff --git a/include/openpose/wrapper/wrapperStructPose.hpp b/include/openpose/wrapper/wrapperStructPose.hpp index a8844d84..634162fc 100644 --- a/include/openpose/wrapper/wrapperStructPose.hpp +++ b/include/openpose/wrapper/wrapperStructPose.hpp @@ -123,13 +123,17 @@ namespace op * It has the recommended and default values we recommend for each element of the struct. * Since all the elements of the struct are public, they can also be manually filled. */ - WrapperStructPose(const Point& netInputSize = Point{656, 368}, const Point& outputSize = Point{1280, 720}, - const ScaleMode keypointScale = ScaleMode::InputResolution, const int gpuNumber = 1, const int gpuNumberStart = 0, - const int scalesNumber = 1, const float scaleGap = 0.15f, const RenderMode renderMode = RenderMode::None, + WrapperStructPose(const Point& netInputSize = Point{656, 368}, + const Point& outputSize = Point{1280, 720}, + const ScaleMode keypointScale = ScaleMode::InputResolution, + const int gpuNumber = 1, const int gpuNumberStart = 0, const int scalesNumber = 1, + const float scaleGap = 0.15f, const RenderMode renderMode = RenderMode::None, const PoseModel poseModel = PoseModel::COCO_18, const bool blendOriginalFrame = true, - const float alphaKeypoint = POSE_DEFAULT_ALPHA_KEYPOINT, const float alphaHeatMap = POSE_DEFAULT_ALPHA_HEAT_MAP, + const float alphaKeypoint = POSE_DEFAULT_ALPHA_KEYPOINT, + const float alphaHeatMap = POSE_DEFAULT_ALPHA_HEAT_MAP, const int defaultPartToRender = 0, const std::string& modelFolder = "models/", - const std::vector& heatMapTypes = {}, const ScaleMode heatMapScale = ScaleMode::ZeroToOne); + const std::vector& heatMapTypes = {}, + const ScaleMode heatMapScale = ScaleMode::ZeroToOne); }; } diff --git a/src/openpose/core/maximumBase.cpp b/src/openpose/core/maximumBase.cpp new file mode 100644 index 00000000..bf864e49 --- /dev/null +++ b/src/openpose/core/maximumBase.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +namespace op +{ + template + void maximumCpu(T* targetPtr, int* kernelPtr, const T* const sourcePtr, const std::array& targetSize, const std::array& sourceSize) + { + try + { + UNUSED(targetPtr); + UNUSED(kernelPtr); + UNUSED(sourcePtr); + UNUSED(targetSize); + UNUSED(sourceSize); + error("CPU version not completely implemented.", __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template void maximumCpu(float* targetPtr, int* kernelPtr, const float* const sourcePtr, const std::array& targetSize, const std::array& sourceSize); + template void maximumCpu(double* targetPtr, int* kernelPtr, const double* const sourcePtr, const std::array& targetSize, const std::array& sourceSize); +} diff --git a/src/openpose/core/maximumBase.cu b/src/openpose/core/maximumBase.cu new file mode 100644 index 00000000..6d81abd5 --- /dev/null +++ b/src/openpose/core/maximumBase.cu @@ -0,0 +1,104 @@ +#include +#include +#include +#include +#include +#include + +namespace op +{ + template + __global__ void fillTargetPtrPart(T* targetPtrOffsetted, const T* sourcePtrOffsetted, const int sourceIndex, const int x, const int y) + { + targetPtrOffsetted[0] = x; + targetPtrOffsetted[1] = y; + targetPtrOffsetted[2] = sourcePtrOffsetted[sourceIndex]; + } + + // template + // __global__ void fillTargetPtrChannel(T* targetPtrOffsetted, const T* sourcePtrOffsetted, const int width, const int imageOffset) + // { + // const auto sourceThrustPtr = thrust::device_pointer_cast(sourcePtrOffsetted); + // const auto sourceIndexIterator = thrust::max_element(thrust::device, sourceThrustPtr, sourceThrustPtr + imageOffset); + // const auto sourceIndex = sourceIndexIterator - sourceThrustPtr; + // targetPtrOffsetted[0] = sourceIndex % width; + // targetPtrOffsetted[1] = sourceIndex / width; + // targetPtrOffsetted[2] = sourcePtrOffsetted[sourceIndex]; + // } + + // template + // __global__ void fillTargetPtr(T* targetPtr, const T* sourcePtr, const int width, const int imageOffset, const int numberSubparts, const int offsetChannel) + // { + // // get pixel location (x,y) + // const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; + // const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; + // const auto part = y*width + x; + + // // if (0 < x && x < (w-1) && 0 < y && y < (h-1)) + // { + // auto* targetPtrOffsetted = targetPtr + (offsetChannel + part) * numberSubparts; + // const auto* const sourcePtrOffsetted = sourcePtr + (offsetChannel + part) * imageOffset; + // auto sourceThrustPtr = thrust::device_pointer_cast(sourcePtrOffsetted); + // const auto sourceIndexIterator = thrust::max_element(thrust::device, sourceThrustPtr, sourceThrustPtr + imageOffset); + // const auto sourceIndex = sourceIndexIterator - sourceThrustPtr; + // targetPtrOffsetted[0] = sourceIndex % width; + // targetPtrOffsetted[1] = sourceIndex / width; + // targetPtrOffsetted[2] = sourcePtrOffsetted[sourceIndex]; + // } + // } + + template + void maximumGpu(T* targetPtr, const T* const sourcePtr, const std::array& targetSize, const std::array& sourceSize) + { + try + { + const auto height = sourceSize[2]; + const auto width = sourceSize[3]; + const auto imageOffset = height * width; + const auto num = targetSize[0]; + const auto channels = targetSize[1]; + const auto numberParts = targetSize[2]; + const auto numberSubparts = targetSize[3]; + + // log("sourceSize[0]: " + std::to_string(sourceSize[0])); // = 1 + // log("sourceSize[1]: " + std::to_string(sourceSize[1])); // = #body parts + bck = 22 (hands) or 71 (face) + // log("sourceSize[2]: " + std::to_string(sourceSize[2])); // = 368 = height + // log("sourceSize[3]: " + std::to_string(sourceSize[3])); // = 368 = width + // log("targetSize[0]: " + std::to_string(targetSize[0])); // = 1 + // log("targetSize[1]: " + std::to_string(targetSize[1])); // = 1 + // log("targetSize[2]: " + std::to_string(targetSize[2])); // = 21(hands) or 70 (face) + // log("targetSize[3]: " + std::to_string(targetSize[3])); // = 3 = [x, y, score] + // log(" "); + for (auto n = 0; n < num; n++) + { + for (auto c = 0; c < channels; c++) + { + // // Parameters + const auto offsetChannel = (n * channels + c); + for (auto part = 0; part < numberParts; part++) + { + auto* targetPtrOffsetted = targetPtr + (offsetChannel + part) * numberSubparts; + const auto* const sourcePtrOffsetted = sourcePtr + (offsetChannel + part) * imageOffset; + // Option a - 6.3 fps + const auto sourceThrustPtr = thrust::device_pointer_cast(sourcePtrOffsetted); + const auto sourceIndexIterator = thrust::max_element(thrust::device, sourceThrustPtr, sourceThrustPtr + imageOffset); + const auto sourceIndex = sourceIndexIterator - sourceThrustPtr; + fillTargetPtrPart<<<1, 1>>>(targetPtrOffsetted, sourcePtrOffsetted, sourceIndex, sourceIndex % width, sourceIndex / width); + // // Option b - <1 fps + // fillTargetPtrChannel<<<1, 1>>>(targetPtrOffsetted, sourcePtrOffsetted, width, imageOffset); + } + // Option c - 4.9 fps + // fillTargetPtr<<<1, numberParts>>>(targetPtr, sourcePtr, width, imageOffset, numberSubparts, offsetChannel); + } + } + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template void maximumGpu(float* targetPtr, const float* const sourcePtr, const std::array& targetSize, const std::array& sourceSize); + template void maximumGpu(double* targetPtr, const double* const sourcePtr, const std::array& targetSize, const std::array& sourceSize); +} diff --git a/src/openpose/core/maximumCaffe.cpp b/src/openpose/core/maximumCaffe.cpp new file mode 100644 index 00000000..1469bcec --- /dev/null +++ b/src/openpose/core/maximumCaffe.cpp @@ -0,0 +1,119 @@ +#ifdef USE_CAFFE +#include +#include +#include +#include + +namespace op +{ + template + MaximumCaffe::MaximumCaffe() + { + } + + template + void MaximumCaffe::LayerSetUp(const std::vector*>& bottom, const std::vector*>& top) + { + try + { + if (top.size() != 1) + error("top.size() != 1", __LINE__, __FUNCTION__, __FILE__); + if (bottom.size() != 1) + error("bottom.size() != 1", __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template + void MaximumCaffe::Reshape(const std::vector*>& bottom, const std::vector*>& top) + { + try + { + auto bottomBlob = bottom.at(0); + auto topBlob = top.at(0); + + // Bottom shape + std::vector bottomShape = bottomBlob->shape(); + + // Top shape + std::vector topShape{bottomShape}; + topShape[1] = 1; // Unnecessary + topShape[2] = bottomShape[1]-1; // Number parts + bck - 1 + topShape[3] = 3; // X, Y, score + topBlob->Reshape(topShape); + + // Array sizes + mTopSize = std::array{topBlob->shape(0), topBlob->shape(1), topBlob->shape(2), topBlob->shape(3)}; + mBottomSize = std::array{bottomBlob->shape(0), bottomBlob->shape(1), bottomBlob->shape(2), bottomBlob->shape(3)}; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template + void MaximumCaffe::Forward_cpu(const std::vector*>& bottom, const std::vector*>& top) + { + try + { + maximumGpu(top.at(0)->mutable_cpu_data(), bottom.at(0)->cpu_data(), mTopSize, mBottomSize); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template + void MaximumCaffe::Forward_gpu(const std::vector*>& bottom, const std::vector*>& top) + { + try + { + maximumGpu(top.at(0)->mutable_gpu_data(), bottom.at(0)->gpu_data(), mTopSize, mBottomSize); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template + void MaximumCaffe::Backward_cpu(const std::vector*>& top, const std::vector& propagate_down, const std::vector*>& bottom) + { + try + { + UNUSED(top); + UNUSED(propagate_down); + UNUSED(bottom); + NOT_IMPLEMENTED; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template + void MaximumCaffe::Backward_gpu(const std::vector*>& top, const std::vector& propagate_down, const std::vector*>& bottom) + { + try + { + UNUSED(top); + UNUSED(propagate_down); + UNUSED(bottom); + NOT_IMPLEMENTED; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + INSTANTIATE_CLASS(MaximumCaffe); +} + +#endif diff --git a/src/openpose/core/nmsBase.cu b/src/openpose/core/nmsBase.cu index 40a04c32..1ac3da4b 100644 --- a/src/openpose/core/nmsBase.cu +++ b/src/openpose/core/nmsBase.cu @@ -1,5 +1,5 @@ -#include #include +#include #include #include #include diff --git a/src/openpose/core/nmsCaffe.cpp b/src/openpose/core/nmsCaffe.cpp index 06d01bf8..309588e0 100644 --- a/src/openpose/core/nmsCaffe.cpp +++ b/src/openpose/core/nmsCaffe.cpp @@ -28,7 +28,7 @@ namespace op } template - void NmsCaffe::Reshape(const std::vector*>& bottom, const std::vector*>& top, const int maxPeaks, const int numberParts) + void NmsCaffe::Reshape(const std::vector*>& bottom, const std::vector*>& top, const int maxPeaks) { try { @@ -39,11 +39,11 @@ namespace op std::vector bottomShape = bottomBlob->shape(); // Top shape - std::vector top_shape{bottomShape}; - top_shape[1] = numberParts; - top_shape[2] = maxPeaks+1; // # maxPeaks + 1 - top_shape[3] = 3; // X, Y, score - topBlob->Reshape(top_shape); + std::vector topShape{bottomShape}; + topShape[1] = bottomShape[1]-1; // Number parts + bck - 1 + topShape[2] = maxPeaks+1; // # maxPeaks + 1 + topShape[3] = 3; // X, Y, score + topBlob->Reshape(topShape); mKernelBlob.Reshape(bottomShape); // Array sizes diff --git a/src/openpose/face/faceExtractor.cpp b/src/openpose/face/faceExtractor.cpp index 68e281e8..bce0b5cd 100644 --- a/src/openpose/face/faceExtractor.cpp +++ b/src/openpose/face/faceExtractor.cpp @@ -16,7 +16,7 @@ namespace op spNet{std::make_shared(std::array{1, 3, mNetOutputSize.y, mNetOutputSize.x}, modelFolder + FACE_PROTOTXT, modelFolder + FACE_TRAINED_MODEL, gpuId)}, spResizeAndMergeCaffe{std::make_shared>()}, - spNmsCaffe{std::make_shared>()}, + spMaximumCaffe{std::make_shared>()}, mFaceImageCrop{mNetOutputSize.area()*3} { try @@ -24,10 +24,6 @@ namespace op checkE(netOutputSize.x, netInputSize.x, "Net input and output size must be equal.", __LINE__, __FUNCTION__, __FILE__); checkE(netOutputSize.y, netInputSize.y, "Net input and output size must be equal.", __LINE__, __FUNCTION__, __FILE__); checkE(netInputSize.x, netInputSize.y, "Net input size must be squared.", __LINE__, __FUNCTION__, __FILE__); - // Properties - for (auto& property : mProperties) - property = 0.; - mProperties[(int)FaceProperty::NMSThreshold] = FACE_DEFAULT_NMS_THRESHOLD; } catch (const std::exception& e) { @@ -57,7 +53,7 @@ namespace op // Pose extractor blob and layer spPeaksBlob = {std::make_shared>(1,1,1,1)}; - spNmsCaffe->Reshape({spHeatMapsBlob.get()}, {spPeaksBlob.get()}, FACE_MAX_PEAKS, FACE_NUMBER_PARTS+1); + spMaximumCaffe->Reshape({spHeatMapsBlob.get()}, {spPeaksBlob.get()}); cudaCheck(__LINE__, __FUNCTION__, __FILE__); log("Finished initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__); @@ -142,49 +138,28 @@ namespace op #endif // 3. Get peaks by Non-Maximum Suppression - spNmsCaffe->setThreshold((float)get(FaceProperty::NMSThreshold)); #ifndef CPU_ONLY - spNmsCaffe->Forward_gpu({spHeatMapsBlob.get()}, {spPeaksBlob.get()}); + spMaximumCaffe->Forward_gpu({spHeatMapsBlob.get()}, {spPeaksBlob.get()}); cudaCheck(__LINE__, __FUNCTION__, __FILE__); #else - spNmsCaffe->Forward_cpu({spHeatMapsBlob.get()}, {spPeaksBlob.get()}); + spMaximumCaffe->Forward_cpu({spHeatMapsBlob.get()}, {spPeaksBlob.get()}); #endif const auto* facePeaksPtr = spPeaksBlob->mutable_cpu_data(); - const auto facePeaksOffset = (FACE_MAX_PEAKS+1) * 3; - for (auto part = 0 ; part < mFaceKeypoints.getSize(1) ; part++) { - // Get max peak - const int numPeaks = intRound(facePeaksPtr[facePeaksOffset*part]); - auto maxScore = -1.f; - auto maxPeak = -1; - for (auto peak = 0 ; peak < numPeaks ; peak++) - { - const auto xyIndex = facePeaksOffset * part + (1 + peak) * 3; - const auto score = facePeaksPtr[xyIndex + 2]; - if (score > maxScore) - { - maxScore = score; - maxPeak = peak; - } - } - // Fill face keypoints - if (maxPeak >= 0) - { - const auto xyIndex = facePeaksOffset * part + (1 + maxPeak) * 3; - const auto x = facePeaksPtr[xyIndex]; - const auto y = facePeaksPtr[xyIndex + 1]; - const auto score = facePeaksPtr[xyIndex + 2]; - const auto baseIndex = mFaceKeypoints.getSize(2) * (person * mFaceKeypoints.getSize(1) + part); - mFaceKeypoints[baseIndex] = (float)(scaleInputToOutput * (Mscaling.at(0,0) * x - + Mscaling.at(0,1) * y - + Mscaling.at(0,2))); - mFaceKeypoints[baseIndex+1] = (float)(scaleInputToOutput * (Mscaling.at(1,0) * x - + Mscaling.at(1,1) * y - + Mscaling.at(1,2))); - mFaceKeypoints[baseIndex+2] = score; - } + const auto xyIndex = part * mFaceKeypoints.getSize(2); + const auto x = facePeaksPtr[xyIndex]; + const auto y = facePeaksPtr[xyIndex + 1]; + const auto score = facePeaksPtr[xyIndex + 2]; + const auto baseIndex = mFaceKeypoints.getSize(2) * (part + person * mFaceKeypoints.getSize(1)); + mFaceKeypoints[baseIndex] = (float)(scaleInputToOutput * (Mscaling.at(0,0) * x + + Mscaling.at(0,1) * y + + Mscaling.at(0,2))); + mFaceKeypoints[baseIndex+1] = (float)(scaleInputToOutput * (Mscaling.at(1,0) * x + + Mscaling.at(1,1) * y + + Mscaling.at(1,2))); + mFaceKeypoints[baseIndex+2] = score; } } } @@ -214,43 +189,6 @@ namespace op } } - double FaceExtractor::get(const FaceProperty property) const - { - try - { - return mProperties.at((int)property); - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return 0.; - } - } - - void FaceExtractor::set(const FaceProperty property, const double value) - { - try - { - mProperties.at((int)property) = {value}; - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } - - void FaceExtractor::increase(const FaceProperty property, const double value) - { - try - { - mProperties[(int)property] = mProperties.at((int)property) + value; - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } - void FaceExtractor::checkThread() const { try diff --git a/src/openpose/filestream/fileStream.cpp b/src/openpose/filestream/fileStream.cpp index 5e6d176c..ac08a8f3 100644 --- a/src/openpose/filestream/fileStream.cpp +++ b/src/openpose/filestream/fileStream.cpp @@ -1,5 +1,9 @@ +#include // std::ifstream #include // cv::imread #include +#include +#include +#include #include #include @@ -154,21 +158,20 @@ namespace op for (const auto& keypointPair : keypointVector) if (!keypointPair.first.empty() && keypointPair.first.getNumberDimensions() != 3 ) error("keypointVector.getNumberDimensions() != 3.", __LINE__, __FUNCTION__, __FILE__); - // Record frame on desired path JsonOfstream jsonOfstream{fileName, humanReadable}; jsonOfstream.objectOpen(); - // Version jsonOfstream.key("version"); jsonOfstream.plainText("1.0"); jsonOfstream.comma(); - // Bodies jsonOfstream.key("people"); jsonOfstream.arrayOpen(); - - const auto numberPeople = (keypointVector.size() > 0 ? keypointVector[0].first.getSize(0) : 0); + // Ger max numberPeople + auto numberPeople = 0; + for (auto vectorIndex = 0 ; vectorIndex < keypointVector.size() ; vectorIndex++) + numberPeople = fastMax(numberPeople, keypointVector[vectorIndex].first.getSize(0)); for (auto person = 0 ; person < numberPeople ; person++) { jsonOfstream.objectOpen(); @@ -202,8 +205,9 @@ namespace op jsonOfstream.enter(); } } + // Close array jsonOfstream.arrayClose(); - + // Close object jsonOfstream.objectClose(); } catch (const std::exception& e) @@ -240,4 +244,44 @@ namespace op return cv::Mat{}; } } + + std::vector, 2>> loadHandDetectorTxt(const std::string& txtFilePath) + { + try + { + std::vector, 2>> handRectangles; + + std::string line; + std::ifstream jsonFile{txtFilePath}; + if (jsonFile.is_open()) + { + while (std::getline(jsonFile, line)) + { + const auto splittedStrings = splitString(line, " "); + std::vector splittedInts; + for (auto splittedString : splittedStrings) + splittedInts.emplace_back(std::stof(splittedString)); + if (splittedInts.size() != 4) + error("splittedInts.size() != 4, but splittedInts.size() = " + + std::to_string(splittedInts.size()) + ".", __LINE__, __FUNCTION__, __FILE__); + const Rectangle handRectangleZero; + const Rectangle handRectangle{splittedInts[0], splittedInts[1], splittedInts[2], splittedInts[3]}; + if (getFileNameNoExtension(txtFilePath).back() == 'l') + handRectangles.emplace_back(std::array, 2>{handRectangle, handRectangleZero}); + else + handRectangles.emplace_back(std::array, 2>{handRectangleZero, handRectangle}); + } + jsonFile.close(); + } + else + error("Unable to open file " + txtFilePath + ".", __LINE__, __FUNCTION__, __FILE__); + + return handRectangles; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return {}; + } + } } diff --git a/src/openpose/hand/defineTemplates.cpp b/src/openpose/hand/defineTemplates.cpp index b8dd7aeb..ec0ed8b0 100644 --- a/src/openpose/hand/defineTemplates.cpp +++ b/src/openpose/hand/defineTemplates.cpp @@ -3,6 +3,7 @@ namespace op { DEFINE_TEMPLATE_DATUM(WHandDetector); + DEFINE_TEMPLATE_DATUM(WHandDetectorFromTxt); DEFINE_TEMPLATE_DATUM(WHandDetectorTracking); DEFINE_TEMPLATE_DATUM(WHandDetectorUpdate); DEFINE_TEMPLATE_DATUM(WHandExtractor); diff --git a/src/openpose/hand/handDetectorFromTxt.cpp b/src/openpose/hand/handDetectorFromTxt.cpp new file mode 100644 index 00000000..a1aaad63 --- /dev/null +++ b/src/openpose/hand/handDetectorFromTxt.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include + +namespace op +{ + std::vector getTxtPathsOnDirectory(const std::string& txtDirectoryPath) + { + try + { + // Get files on directory with JSON extension + const auto txtPaths = getFilesOnDirectory(txtDirectoryPath, ".txt"); + // Check #files > 0 + if (txtPaths.empty()) + error("No txt files were found on " + txtDirectoryPath, __LINE__, __FUNCTION__, __FILE__); + // Return file names + return txtPaths; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return {}; + } + } + + HandDetectorFromTxt::HandDetectorFromTxt(const std::string& txtDirectoryPath) : + mTxtDirectoryPath{txtDirectoryPath}, + mFilePaths{getTxtPathsOnDirectory(txtDirectoryPath)}, + mFrameNameCounter{0} + { + } + + std::vector, 2>> HandDetectorFromTxt::detectHands() + { + try + { + return loadHandDetectorTxt(mFilePaths.at(mFrameNameCounter++)); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return std::vector, 2>>{}; + } + } +} diff --git a/src/openpose/pose/poseExtractorCaffe.cpp b/src/openpose/pose/poseExtractorCaffe.cpp index 11f9b91a..c8a37c98 100644 --- a/src/openpose/pose/poseExtractorCaffe.cpp +++ b/src/openpose/pose/poseExtractorCaffe.cpp @@ -56,7 +56,7 @@ namespace op // Pose extractor blob and layer spPeaksBlob = {std::make_shared>(1,1,1,1)}; - spNmsCaffe->Reshape({spHeatMapsBlob.get()}, {spPeaksBlob.get()}, POSE_MAX_PEAKS[(int)mPoseModel], POSE_NUMBER_BODY_PARTS[(int)mPoseModel]); + spNmsCaffe->Reshape({spHeatMapsBlob.get()}, {spPeaksBlob.get()}, POSE_MAX_PEAKS[(int)mPoseModel]); cudaCheck(__LINE__, __FUNCTION__, __FILE__); // Pose extractor blob and layer diff --git a/src/openpose/utilities/string.cpp b/src/openpose/utilities/string.cpp index d5c6aee4..577c4c7a 100644 --- a/src/openpose/utilities/string.cpp +++ b/src/openpose/utilities/string.cpp @@ -34,6 +34,29 @@ namespace op } } + std::vector splitString(const std::string& stringToSplit, const std::string& delimiter) + { + try + { + std::vector result; + size_t pos = 0; + auto stringToSplitAux = stringToSplit; + while ((pos = stringToSplitAux.find(delimiter)) != std::string::npos) + { + result.emplace_back(stringToSplitAux.substr(0, pos)); + stringToSplitAux.erase(0, pos + delimiter.length()); + } + result.emplace_back(stringToSplitAux); + return result; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return {}; + } + } + + // Signed template std::string toFixedLengthString(const char number, const unsigned long long stringLength); template std::string toFixedLengthString(const signed char number, const unsigned long long stringLength); diff --git a/src/openpose/wrapper/wrapperStructPose.cpp b/src/openpose/wrapper/wrapperStructPose.cpp index 6addd113..9fc823a4 100644 --- a/src/openpose/wrapper/wrapperStructPose.cpp +++ b/src/openpose/wrapper/wrapperStructPose.cpp @@ -2,10 +2,14 @@ namespace op { - WrapperStructPose::WrapperStructPose(const Point& netInputSize_, const Point& outputSize_, const ScaleMode keypointScale_, const int gpuNumber_, - const int gpuNumberStart_, const int scalesNumber_, const float scaleGap_, const RenderMode renderMode_, - const PoseModel poseModel_, const bool blendOriginalFrame_, const float alphaKeypoint_, const float alphaHeatMap_, - const int defaultPartToRender_, const std::string& modelFolder_, const std::vector& heatMapTypes_, + WrapperStructPose::WrapperStructPose(const Point& netInputSize_, const Point& outputSize_, + const ScaleMode keypointScale_, const int gpuNumber_, + const int gpuNumberStart_, const int scalesNumber_, const float scaleGap_, + const RenderMode renderMode_, const PoseModel poseModel_, + const bool blendOriginalFrame_, const float alphaKeypoint_, + const float alphaHeatMap_, const int defaultPartToRender_, + const std::string& modelFolder_, + const std::vector& heatMapTypes_, const ScaleMode heatMapScale_) : netInputSize{netInputSize_}, outputSize{outputSize_},