diff --git a/doc/UML/1_0_0rc1/UML.mdj b/doc/UML/1_0_0rc1/UML.mdj index c592aed2..59660927 100755 --- a/doc/UML/1_0_0rc1/UML.mdj +++ b/doc/UML/1_0_0rc1/UML.mdj @@ -5917,7 +5917,7 @@ "_parent": { "$ref": "AAAAAAFbHDZmRV10zeE=" }, - "name": "scalePose", + "name": "scaleKeyPoints", "ownedElements": [ { "_type": "UMLDependency", @@ -6005,7 +6005,7 @@ "_parent": { "$ref": "AAAAAAFbHDZmRV10zeE=" }, - "name": "wArrayScaler.hpp", + "name": "wKeyPointScaler.hpp", "ownedElements": [ { "_type": "UMLDependency", @@ -6076,7 +6076,7 @@ "_parent": { "$ref": "AAAAAAFbHDZmRV10zeE=" }, - "name": "ArrayScaler", + "name": "KeyPointScaler", "ownedElements": [ { "_type": "UMLDependency", @@ -13344,7 +13344,7 @@ "height": 13, "autoResize": false, "underline": false, - "text": "scalePose", + "text": "scaleKeyPoints", "horizontalAlignment": 2, "verticalAlignment": 5, "wordWrap": false @@ -13635,7 +13635,7 @@ "height": 13, "autoResize": false, "underline": false, - "text": "wArrayScaler.hpp", + "text": "wKeyPointScaler.hpp", "horizontalAlignment": 2, "verticalAlignment": 5, "wordWrap": false @@ -14633,7 +14633,7 @@ "height": 13, "autoResize": false, "underline": false, - "text": "ArrayScaler", + "text": "KeyPointScaler", "horizontalAlignment": 2, "verticalAlignment": 5, "wordWrap": false diff --git a/doc/UML/1_0_0rc1/UML.pdf b/doc/UML/1_0_0rc1/UML.pdf index 60481c21..0c551505 100644 Binary files a/doc/UML/1_0_0rc1/UML.pdf and b/doc/UML/1_0_0rc1/UML.pdf differ diff --git a/examples/tutorial_pose/1_extract_from_image.cpp b/examples/tutorial_pose/1_extract_from_image.cpp index 73f98722..8533d4c8 100644 --- a/examples/tutorial_pose/1_extract_from_image.cpp +++ b/examples/tutorial_pose/1_extract_from_image.cpp @@ -117,11 +117,11 @@ int openPoseTutorialPose1() double scaleInputToOutput; op::Array outputArray; std::tie(scaleInputToOutput, outputArray) = cvMatToOpOutput.format(inputImage); - // Step 3 - Estimate pose + // Step 3 - Estimate poseKeyPoints poseExtractorCaffe.forwardPass(netInputArray, inputImage.size()); - const auto pose = poseExtractorCaffe.getPose(); - // Step 4 - Render pose - poseRenderer.renderPose(outputArray, pose); + const auto poseKeyPoints = poseExtractorCaffe.getPoseKeyPoints(); + // Step 4 - Render poseKeyPoints + poseRenderer.renderPose(outputArray, poseKeyPoints); // Step 5 - OpenPose output format to cv::Mat auto outputImage = opOutputToCvMat.formatToCvMat(outputArray); diff --git a/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp b/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp index fadc51e6..b4de4cfa 100644 --- a/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp +++ b/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp @@ -121,12 +121,12 @@ int openPoseTutorialPose2() double scaleInputToOutput; op::Array outputArray; std::tie(scaleInputToOutput, outputArray) = cvMatToOpOutput.format(inputImage); - // Step 3 - Estimate pose + // Step 3 - Estimate poseKeyPoints poseExtractorPtr->forwardPass(netInputArray, inputImage.size()); - const auto pose = poseExtractorPtr->getPose(); + const auto poseKeyPoints = poseExtractorPtr->getPoseKeyPoints(); const auto scaleNetToOutput = poseExtractorPtr->getScaleNetToOutput(); // Step 4 - Render pose - poseRenderer.renderPose(outputArray, pose, scaleNetToOutput); + poseRenderer.renderPose(outputArray, poseKeyPoints, scaleNetToOutput); // Step 5 - OpenPose output format to cv::Mat auto outputImage = opOutputToCvMat.formatToCvMat(outputArray); diff --git a/examples/tutorial_thread/3_user_input_processing_and_output.cpp b/examples/tutorial_thread/3_user_input_processing_and_output.cpp index 3da831a0..ebb21608 100644 --- a/examples/tutorial_thread/3_user_input_processing_and_output.cpp +++ b/examples/tutorial_thread/3_user_input_processing_and_output.cpp @@ -114,7 +114,7 @@ public: { // User's post-processing (after OpenPose processing & before OpenPose outputs) here // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.pose: Array with the estimated pose + // datum.poseKeyPoints: Array with the estimated pose try { if (datumsPtr != nullptr && !datumsPtr->empty()) @@ -142,7 +142,7 @@ public: { // User's displaying/saving/other processing here // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.pose: Array with the estimated pose + // datum.poseKeyPoints: Array with the estimated pose if (datumsPtr != nullptr && !datumsPtr->empty()) { cv::imshow("User worker GUI", datumsPtr->at(0).cvOutputData); diff --git a/examples/tutorial_thread/4_user_input_processing_output_and_datum.cpp b/examples/tutorial_thread/4_user_input_processing_output_and_datum.cpp index 737620c0..c5d2e081 100644 --- a/examples/tutorial_thread/4_user_input_processing_output_and_datum.cpp +++ b/examples/tutorial_thread/4_user_input_processing_output_and_datum.cpp @@ -126,7 +126,7 @@ public: { // User's post-processing (after OpenPose processing & before OpenPose outputs) here // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.pose: Array with the estimated pose + // datum.poseKeyPoints: Array with the estimated pose try { if (datumsPtr != nullptr && !datumsPtr->empty()) @@ -154,7 +154,7 @@ public: { // User's displaying/saving/other processing here // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.pose: Array with the estimated pose + // datum.poseKeyPoints: Array with the estimated pose if (datumsPtr != nullptr && !datumsPtr->empty()) { cv::imshow("User worker GUI", datumsPtr->at(0).cvOutputData); diff --git a/examples/tutorial_wrapper/1_user_asynchronous.cpp b/examples/tutorial_wrapper/1_user_asynchronous.cpp index 8841e843..509afa8d 100644 --- a/examples/tutorial_wrapper/1_user_asynchronous.cpp +++ b/examples/tutorial_wrapper/1_user_asynchronous.cpp @@ -176,7 +176,7 @@ public: { // User's displaying/saving/other processing here // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.pose: Array with the estimated pose + // datum.poseKeyPoints: Array with the estimated pose if (datumsPtr != nullptr && !datumsPtr->empty()) { cv::imshow("User worker GUI", datumsPtr->at(0).cvOutputData); diff --git a/examples/tutorial_wrapper/2_user_synchronous.cpp b/examples/tutorial_wrapper/2_user_synchronous.cpp index d37ff4f6..bafabf1b 100644 --- a/examples/tutorial_wrapper/2_user_synchronous.cpp +++ b/examples/tutorial_wrapper/2_user_synchronous.cpp @@ -188,7 +188,7 @@ public: { // User's post-processing (after OpenPose processing & before OpenPose outputs) here // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.pose: Array with the estimated pose + // datum.poseKeyPoints: Array with the estimated pose try { if (datumsPtr != nullptr && !datumsPtr->empty()) @@ -216,7 +216,7 @@ public: { // User's displaying/saving/other processing here // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.pose: Array with the estimated pose + // datum.poseKeyPoints: Array with the estimated pose if (datumsPtr != nullptr && !datumsPtr->empty()) { cv::imshow("User worker GUI", datumsPtr->at(0).cvOutputData); diff --git a/include/openpose/core/arrayScaler.hpp b/include/openpose/core/arrayScaler.hpp deleted file mode 100644 index 1bfe9803..00000000 --- a/include/openpose/core/arrayScaler.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef OPENPOSE__CORE__ARRAYS_SCALER_HPP -#define OPENPOSE__CORE__ARRAYS_SCALER_HPP - -#include -#include "array.hpp" -#include "enumClasses.hpp" - -namespace op -{ - class ArrayScaler - { - public: - explicit ArrayScaler(const ScaleMode scalePose); - - void scale(Array& array, const double scaleInputToOutput, const double scaleNetToOutput, const cv::Size& producerSize) const; - - void scale(std::vector>& arrays, const double scaleInputToOutput, const double scaleNetToOutput, const cv::Size& producerSize) const; - - private: - const ScaleMode mScaleMode; - }; -} - -#endif // OPENPOSE__CORE__ARRAYS_SCALER_HPP diff --git a/include/openpose/core/datum.hpp b/include/openpose/core/datum.hpp index 7d247058..22e558f1 100644 --- a/include/openpose/core/datum.hpp +++ b/include/openpose/core/datum.hpp @@ -52,15 +52,15 @@ namespace op // -------------------------------------------------- Resulting Array data parameters -------------------------------------------------- // /** - * Pose (x,y,score) locations for each person in the image. + * Body pose (x,y,score) locations for each person in the image. * It has been resized to the desired output resolution (e.g. `resolution` flag in the demo). * If outputData is empty, then cvOutputData will also be empty. * Size: #people x #body parts (e.g. 18 for COCO or 15 for MPI) x 3 ((x,y) coordinates + score) */ + Array poseKeyPoints; - Array pose; /** - * Pose heatmaps (body parts, background and PAFs) for the whole image. + * Body pose heatmaps (body parts, background and/or PAFs) for the whole image. * This parameters is by default empty and disabled for performance. Each group (body parts, background and PAFs) can be individually enabled. * #heatmaps = #body parts (if enabled) + 1 (if background enabled) + 2 x #PAFs (if enabled). Each PAF has 2 consecutive channels, one for x- and one for y-coordinates. * Order heatmaps: body parts + background (as appears in POSE_BODY_PART_MAPPING) + (x,y) channel of each PAF (sorted as appears in POSE_BODY_PART_PAIRS). See `pose/poseParameters.hpp`. @@ -71,10 +71,17 @@ namespace op /** * Experimental (NOT IMPLEMENTED YET) - * Hands code is in development phase. Not included in this version. - * Size: 2 (right and left hand) x #hand parts (21) x 3 ((x,y) coordinates + score) + * Face code is in development phase. Not included in this version. + * Size: #people's faces to render x #face parts (71) x 3 ((x,y) coordinates + score) */ - Array hands; + Array faceKeyPoints; + + /** + * Experimental (NOT IMPLEMENTED YET) + * Hands code is in development phase. Not included in this version. + * Size: #people's hands to render x 2 (right and left hand) x #hand parts (21) x 3 ((x,y) coordinates + score) + */ + Array handKeyPoints; // -------------------------------------------------- Other parameters -------------------------------------------------- // double scaleInputToOutput; /**< Scale ratio between the input Datum::cvInputData and the output Datum::cvOutputData. */ diff --git a/include/openpose/core/headers.hpp b/include/openpose/core/headers.hpp index fe57221d..1075cf7c 100644 --- a/include/openpose/core/headers.hpp +++ b/include/openpose/core/headers.hpp @@ -3,11 +3,11 @@ // core module #include "array.hpp" -#include "arrayScaler.hpp" #include "cvMatToOpInput.hpp" #include "cvMatToOpOutput.hpp" #include "datum.hpp" #include "enumClasses.hpp" +#include "keyPointScaler.hpp" #include "net.hpp" #include "netCaffe.hpp" #include "nmsBase.hpp" @@ -16,10 +16,10 @@ #include "renderer.hpp" #include "resizeAndMergeBase.hpp" #include "resizeAndMergeCaffe.hpp" -#include "scalePose.hpp" -#include "wArrayScaler.hpp" +#include "scaleKeyPoints.hpp" #include "wCvMatToOpInput.hpp" #include "wCvMatToOpOutput.hpp" +#include "wKeyPointScaler.hpp" #include "wOpOutputToCvMat.hpp" #endif // OPENPOSE__CORE__HEADERS_HPP diff --git a/include/openpose/core/keyPointScaler.hpp b/include/openpose/core/keyPointScaler.hpp new file mode 100644 index 00000000..27752d23 --- /dev/null +++ b/include/openpose/core/keyPointScaler.hpp @@ -0,0 +1,24 @@ +#ifndef OPENPOSE__CORE__KEY_POINT_SCALER_HPP +#define OPENPOSE__CORE__KEY_POINT_SCALER_HPP + +#include +#include "array.hpp" +#include "enumClasses.hpp" + +namespace op +{ + class KeyPointScaler + { + public: + explicit KeyPointScaler(const ScaleMode scaleMode); + + void scale(Array& arrayToScale, const double scaleInputToOutput, const double scaleNetToOutput, const cv::Size& producerSize) const; + + void scale(std::vector>& arraysToScale, const double scaleInputToOutput, const double scaleNetToOutput, const cv::Size& producerSize) const; + + private: + const ScaleMode mScaleMode; + }; +} + +#endif // OPENPOSE__CORE__KEY_POINT_SCALER_HPP diff --git a/include/openpose/core/scaleKeyPoints.hpp b/include/openpose/core/scaleKeyPoints.hpp new file mode 100644 index 00000000..9e193b1e --- /dev/null +++ b/include/openpose/core/scaleKeyPoints.hpp @@ -0,0 +1,15 @@ +#ifndef OPENPOSE__CORE__SCALE_KEY_POINTS_HPP +#define OPENPOSE__CORE__SCALE_KEY_POINTS_HPP + +#include "array.hpp" + +namespace op +{ + void scaleKeyPoints(Array& keyPoints, const double scale); + + void scaleKeyPoints(Array& keyPoints, const double scaleX, const double scaleY); + + void scaleKeyPoints(Array& keyPoints, const double scaleX, const double scaleY, const double offsetX, const double offsetY); +} + +#endif // OPENPOSE__CORE__SCALE_KEY_POINTS_HPP diff --git a/include/openpose/core/scalePose.hpp b/include/openpose/core/scalePose.hpp deleted file mode 100644 index 9ad9fbed..00000000 --- a/include/openpose/core/scalePose.hpp +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef OPENPOSE__CORE__SCALE_POSE_HPP -#define OPENPOSE__CORE__SCALE_POSE_HPP - -#include "array.hpp" - -namespace op -{ - void scalePose(Array& pose, const double scale); - - void scalePose(Array& pose, const double scaleX, const double scaleY); - - void scalePose(Array& pose, const double scaleX, const double scaleY, const double offsetX, const double offsetY); -} - -#endif // OPENPOSE__CORE__SCALE_POSE_HPP diff --git a/include/openpose/core/wArrayScaler.hpp b/include/openpose/core/wKeyPointScaler.hpp similarity index 60% rename from include/openpose/core/wArrayScaler.hpp rename to include/openpose/core/wKeyPointScaler.hpp index 5811e112..3d8e65a1 100644 --- a/include/openpose/core/wArrayScaler.hpp +++ b/include/openpose/core/wKeyPointScaler.hpp @@ -1,24 +1,24 @@ -#ifndef OPENPOSE__CORE__W_ARRAYS_SCALER_HPP -#define OPENPOSE__CORE__W_ARRAYS_SCALER_HPP +#ifndef OPENPOSE__CORE__W_KEY_POINT_SCALER_HPP +#define OPENPOSE__CORE__W_KEY_POINT_SCALER_HPP #include "../thread/worker.hpp" -#include "arrayScaler.hpp" -#include "scalePose.hpp" +#include "keyPointScaler.hpp" +#include "scaleKeyPoints.hpp" namespace op { template - class WArrayScaler : public Worker + class WKeyPointScaler : public Worker { public: - explicit WArrayScaler(const std::shared_ptr& arrayScaler); + explicit WKeyPointScaler(const std::shared_ptr& keyPointScaler); void initializationOnThread(); void work(TDatums& tDatums); private: - std::shared_ptr spArrayScaler; + std::shared_ptr spKeyPointScaler; }; } @@ -34,18 +34,18 @@ namespace op namespace op { template - WArrayScaler::WArrayScaler(const std::shared_ptr& arrayScaler) : - spArrayScaler{arrayScaler} + WKeyPointScaler::WKeyPointScaler(const std::shared_ptr& keyPointScaler) : + spKeyPointScaler{keyPointScaler} { } template - void WArrayScaler::initializationOnThread() + void WKeyPointScaler::initializationOnThread() { } template - void WArrayScaler::work(TDatums& tDatums) + void WKeyPointScaler::work(TDatums& tDatums) { try { @@ -58,8 +58,8 @@ namespace op // Rescale pose data for (auto& tDatum : *tDatums) { - std::vector> arrays{tDatum.pose, tDatum.hands}; - spArrayScaler->scale(arrays, tDatum.scaleInputToOutput, tDatum.scaleNetToOutput, tDatum.cvInputData.size()); + std::vector> arraysToScale{tDatum.poseKeyPoints, tDatum.handKeyPoints, tDatum.faceKeyPoints}; + spKeyPointScaler->scale(arraysToScale, tDatum.scaleInputToOutput, tDatum.scaleNetToOutput, tDatum.cvInputData.size()); } // Profiling speed Profiler::timerEnd(profilerKey); @@ -76,7 +76,7 @@ namespace op } } - COMPILE_TEMPLATE_DATUM(WArrayScaler); + COMPILE_TEMPLATE_DATUM(WKeyPointScaler); } -#endif // OPENPOSE__CORE__W_ARRAYS_SCALER_HPP +#endif // OPENPOSE__CORE__W_KEY_POINT_SCALER_HPP diff --git a/include/openpose/experimental/face/enumClasses.hpp b/include/openpose/experimental/face/enumClasses.hpp new file mode 100644 index 00000000..be06948e --- /dev/null +++ b/include/openpose/experimental/face/enumClasses.hpp @@ -0,0 +1,14 @@ +#ifndef OPENPOSE__FACE__ENUM_CLASSES_HPP +#define OPENPOSE__FACE__ENUM_CLASSES_HPP + +namespace op +{ + enum class FaceProperty : bool + { + NMSThreshold = 0, + Size, + }; +} + +#endif // OPENPOSE__FACE__ENUM_CLASSES_HPP + diff --git a/include/openpose/experimental/face/faceExtractor.hpp b/include/openpose/experimental/face/faceExtractor.hpp new file mode 100644 index 00000000..b298313a --- /dev/null +++ b/include/openpose/experimental/face/faceExtractor.hpp @@ -0,0 +1,66 @@ +#ifndef OPENPOSE__FACE__FACE_EXTRACTOR_HPP +#define OPENPOSE__FACE__FACE_EXTRACTOR_HPP + +#include +#include +#include // std::shared_ptr +#include +#include +#include "../../core/array.hpp" +#include "../../core/net.hpp" +#include "../../core/nmsCaffe.hpp" +#include "../../core/resizeAndMergeCaffe.hpp" +#include "../../pose/enumClasses.hpp" +#include "enumClasses.hpp" + +namespace op +{ + namespace experimental + { + class FaceExtractor + { + public: + explicit FaceExtractor(const std::string& modelFolder, const int gpuId, const PoseModel poseModel); + + void initializationOnThread(); + + void forwardPass(const Array& poseKeyPoints, const cv::Mat& cvInputData); + + Array getFaceKeyPoints() const; + + double get(const FaceProperty property) const; + + void set(const FaceProperty property, const double value); + + void increase(const FaceProperty property, const double value); + + private: + const cv::Size mNetOutputSize; + const cv::Size mOutputSize; + const unsigned char mNeck; + const unsigned char mNose; + const unsigned char mLEar; + const unsigned char mREar; + const unsigned char mLEye; + const unsigned char mREye; + std::array, (int)FaceProperty::Size> mProperties; + std::shared_ptr spNet; + std::shared_ptr> spResizeAndMergeCaffe; + std::shared_ptr> spNmsCaffe; + Array mFaceImageCrop; + Array mFaceKeyPoints; + float mScaleFace; + // Init with thread + boost::shared_ptr> spCaffeNetOutputBlob; + std::shared_ptr> spHeatMapsBlob; + std::shared_ptr> spPeaksBlob; + std::thread::id mThreadId; + + void checkThread() const; + + DELETE_COPY(FaceExtractor); + }; + } +} + +#endif // OPENPOSE__FACE__FACE_EXTRACTOR_HPP diff --git a/include/openpose/experimental/face/faceParameters.hpp b/include/openpose/experimental/face/faceParameters.hpp new file mode 100644 index 00000000..9c58b057 --- /dev/null +++ b/include/openpose/experimental/face/faceParameters.hpp @@ -0,0 +1,35 @@ +#ifndef OPENPOSE__FACE__FACE_PARAMETERS_HPP +#define OPENPOSE__FACE__FACE_PARAMETERS_HPP + +#include "enumClasses.hpp" +#include "../../pose/poseParameters.hpp" + +namespace op +{ + const unsigned char FACE_MAX_NUMBER_FACE = 1; + + const unsigned char FACE_NUMBER_PARTS = 71; + #define FACE_PAIRS_TO_RENDER {0,1, 1,2, 2,3, 3,4, 4,5, 5,6, 6,7, 7,8, 8,9, 9,10, 10,11, 11,12, 12,13, 13,14, 14,15, 15,16, 17,18, 18,19, 19,20, \ + 20,21, 22,23, 23,24, 24,25, 25,26, 27,28, 28,29, 29,30, 31,32, 32,33, 33,34, 34,35, 36,37, 37,38, 38,39, 39,40, 40,41, \ + 41,36, 42,43, 43,44, 44,45, 45,46, 46,47, 47,42, 48,49, 49,50, 50,51, 51,52, 52,53, 53,54, 54,55, 55,56, 56,57, 57,58, \ + 58,59, 59,48, 60,61, 61,62, 62,63, 63,64, 64,65, 65,66, 66,67, 67,60} + + // Constant Global Parameters + // const unsigned char FACE_MAX_PEOPLE = 1; + + // Constant parameters + const auto FACE_CCN_DECREASE_FACTOR = 8.f; + const unsigned int FACE_MAX_PEAKS = 64u; + const std::string FACE_PROTOTXT{"face/pose_deploy.prototxt"}; + const std::string FACE_TRAINED_MODEL{"face/pose_iter_116000.caffemodel"}; + + // Default Model Parameters + // They might be modified on running time + const auto FACE_DEFAULT_NMS_THRESHOLD = 0.1f; + + // Rendering default parameters + const auto FACE_DEFAULT_ALPHA_FACE = POSE_DEFAULT_ALPHA_POSE; + // const auto FACE_DEFAULT_ALPHA_HEATMAP = POSE_DEFAULT_ALPHA_HEATMAP; +} + +#endif // OPENPOSE__FACE__FACE_PARAMETERS_HPP diff --git a/include/openpose/experimental/face/faceRenderGpu.hpp b/include/openpose/experimental/face/faceRenderGpu.hpp new file mode 100644 index 00000000..80cdaab7 --- /dev/null +++ b/include/openpose/experimental/face/faceRenderGpu.hpp @@ -0,0 +1,12 @@ +#ifndef OPENPOSE__FACE__GPU_FACE_RENDER_HPP +#define OPENPOSE__FACE__GPU_FACE_RENDER_HPP + +#include +#include "faceParameters.hpp" + +namespace op +{ + void renderFaceGpu(float* framePtr, const cv::Size& frameSize, const float* const facePtr, const int numberFace, const float alphaColorToAdd = FACE_DEFAULT_ALPHA_FACE); +} + +#endif // OPENPOSE__FACE__GPU_FACE_RENDER_HPP diff --git a/include/openpose/experimental/face/faceRenderer.hpp b/include/openpose/experimental/face/faceRenderer.hpp new file mode 100644 index 00000000..d366b708 --- /dev/null +++ b/include/openpose/experimental/face/faceRenderer.hpp @@ -0,0 +1,33 @@ +#ifndef OPENPOSE__FACE__FACE_RENDERER_HPP +#define OPENPOSE__FACE__FACE_RENDERER_HPP + +#include +#include "../../core/array.hpp" +#include "../../core/renderer.hpp" +#include "../../thread/worker.hpp" + +namespace op +{ + namespace experimental + { + class FaceRenderer : public Renderer + { + public: + explicit FaceRenderer(const cv::Size& frameSize); + + ~FaceRenderer(); + + void initializationOnThread(); + + void renderFace(Array& outputData, const Array& faceKeyPoints); + + private: + const cv::Size mFrameSize; + float* pGpuFace; // GPU aux memory + + DELETE_COPY(FaceRenderer); + }; + } +} + +#endif // OPENPOSE__FACE__FACE_RENDERER_HPP diff --git a/include/openpose/experimental/face/headers.hpp b/include/openpose/experimental/face/headers.hpp new file mode 100644 index 00000000..4829481b --- /dev/null +++ b/include/openpose/experimental/face/headers.hpp @@ -0,0 +1,13 @@ +#ifndef OPENPOSE__FACE__HEADERS_HPP +#define OPENPOSE__FACE__HEADERS_HPP + +// face module +#include "enumClasses.hpp" +#include "faceExtractor.hpp" +#include "faceParameters.hpp" +#include "faceRenderer.hpp" +#include "faceRenderGpu.hpp" +#include "wFaceExtractor.hpp" +#include "wFaceRenderer.hpp" + +#endif // OPENPOSE__FACE__HEADERS_HPP diff --git a/include/openpose/experimental/face/wFaceExtractor.hpp b/include/openpose/experimental/face/wFaceExtractor.hpp new file mode 100644 index 00000000..dd6c8373 --- /dev/null +++ b/include/openpose/experimental/face/wFaceExtractor.hpp @@ -0,0 +1,91 @@ +#ifndef OPENPOSE__FACE__W_FACE_EXTRACTOR_HPP +#define OPENPOSE__FACE__W_FACE_EXTRACTOR_HPP + +#include // std::shared_ptr +#include "../../thread/worker.hpp" +#include "faceRenderer.hpp" + +namespace op +{ + namespace experimental + { + template + class WFaceExtractor : public Worker + { + public: + explicit WFaceExtractor(const std::shared_ptr& faceExtractor); + + void initializationOnThread(); + + void work(TDatums& tDatums); + + private: + std::shared_ptr spFaceExtractor; + + DELETE_COPY(WFaceExtractor); + }; + } +} + + + + + +// Implementation +#include "../../utilities/errorAndLog.hpp" +#include "../../utilities/macros.hpp" +#include "../../utilities/pointerContainer.hpp" +#include "../../utilities/profiler.hpp" +namespace op +{ + namespace experimental + { + template + WFaceExtractor::WFaceExtractor(const std::shared_ptr& faceExtractor) : + spFaceExtractor{faceExtractor} + { + } + + template + void WFaceExtractor::initializationOnThread() + { + spFaceExtractor->initializationOnThread(); + } + + template + void WFaceExtractor::work(TDatums& tDatums) + { + try + { + if (checkNoNullNorEmpty(tDatums)) + { + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + // Extract people face + for (auto& tDatum : *tDatums) + { + spFaceExtractor->forwardPass(tDatum.poseKeyPoints, tDatum.cvInputData); + tDatum.faceKeyPoints = spFaceExtractor->getFaceKeyPoints(); + } + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 1000); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + } + catch (const std::exception& e) + { + this->stop(); + tDatums = nullptr; + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + COMPILE_TEMPLATE_DATUM(WFaceExtractor); + } +} + +#endif // OPENPOSE__FACE__W_FACE_EXTRACTOR_HPP diff --git a/include/openpose/experimental/face/wFaceRenderer.hpp b/include/openpose/experimental/face/wFaceRenderer.hpp new file mode 100644 index 00000000..9d75ffd2 --- /dev/null +++ b/include/openpose/experimental/face/wFaceRenderer.hpp @@ -0,0 +1,88 @@ +#ifndef OPENPOSE__FACE__W_FACE_RENDERER_HPP +#define OPENPOSE__FACE__W_FACE_RENDERER_HPP + +#include // std::shared_ptr +#include "../../thread/worker.hpp" +#include "faceRenderer.hpp" + +namespace op +{ + namespace experimental + { + template + class WFaceRenderer : public Worker + { + public: + explicit WFaceRenderer(const std::shared_ptr& faceRenderer); + + void initializationOnThread(); + + void work(TDatums& tDatums); + + private: + std::shared_ptr spFaceRenderer; + + DELETE_COPY(WFaceRenderer); + }; + } +} + + + + + +// Implementation +#include "../../utilities/errorAndLog.hpp" +#include "../../utilities/macros.hpp" +#include "../../utilities/pointerContainer.hpp" +#include "../../utilities/profiler.hpp" +namespace op +{ + namespace experimental + { + template + WFaceRenderer::WFaceRenderer(const std::shared_ptr& faceRenderer) : + spFaceRenderer{faceRenderer} + { + } + + template + void WFaceRenderer::initializationOnThread() + { + spFaceRenderer->initializationOnThread(); + } + + template + void WFaceRenderer::work(TDatums& tDatums) + { + try + { + if (checkNoNullNorEmpty(tDatums)) + { + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + // Render people face + for (auto& tDatum : *tDatums) + spFaceRenderer->renderFace(tDatum.outputData, tDatum.faceKeyPoints); + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 1000); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + } + catch (const std::exception& e) + { + this->stop(); + tDatums = nullptr; + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + COMPILE_TEMPLATE_DATUM(WFaceRenderer); + } +} + +#endif // OPENPOSE__FACE__W_FACE_RENDERER_HPP diff --git a/include/openpose/experimental/filestream/wPoseJsonCocoSaver.hpp b/include/openpose/experimental/filestream/wPoseJsonCocoSaver.hpp index 9c75d921..73508ff5 100644 --- a/include/openpose/experimental/filestream/wPoseJsonCocoSaver.hpp +++ b/include/openpose/experimental/filestream/wPoseJsonCocoSaver.hpp @@ -78,7 +78,7 @@ namespace op if (imageId <= 50006) // 3559 images ~ 4 min at 15fps // move this to producer as --frame_last 50006 { // Record json in COCO format if file within desired range of images - spPoseJsonCocoSaver->record(tDatum.pose, imageId); + spPoseJsonCocoSaver->record(tDatum.poseKeyPoints, imageId); // Profiling speed Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 1000); diff --git a/include/openpose/experimental/hand/enumClasses.hpp b/include/openpose/experimental/hand/enumClasses.hpp new file mode 100644 index 00000000..3f77c73e --- /dev/null +++ b/include/openpose/experimental/hand/enumClasses.hpp @@ -0,0 +1,14 @@ +#ifndef OPENPOSE__HAND__ENUM_CLASSES_HPP +#define OPENPOSE__HAND__ENUM_CLASSES_HPP + +namespace op +{ + enum class HandsProperty : bool + { + NMSThreshold = 0, + Size, + }; +} + +#endif // OPENPOSE__HAND__ENUM_CLASSES_HPP + diff --git a/include/openpose/experimental/hands/handsExtractor.hpp b/include/openpose/experimental/hand/handExtractor.hpp similarity index 79% rename from include/openpose/experimental/hands/handsExtractor.hpp rename to include/openpose/experimental/hand/handExtractor.hpp index 587e8399..be5187af 100644 --- a/include/openpose/experimental/hands/handsExtractor.hpp +++ b/include/openpose/experimental/hand/handExtractor.hpp @@ -1,5 +1,5 @@ -#ifndef OPENPOSE__HANDS__HANDS_EXTRACTOR_HPP -#define OPENPOSE__HANDS__HANDS_EXTRACTOR_HPP +#ifndef OPENPOSE__HAND__HAND_EXTRACTOR_HPP +#define OPENPOSE__HAND__HAND_EXTRACTOR_HPP #include #include @@ -17,16 +17,16 @@ namespace op { namespace experimental { - class HandsExtractor + class HandExtractor { public: - explicit HandsExtractor(const std::string& modelFolder, const int gpuId, const PoseModel poseModel); + explicit HandExtractor(const std::string& modelFolder, const int gpuId, const PoseModel poseModel); void initializationOnThread(); - void forwardPass(const Array& pose, const cv::Mat& cvInputData); + void forwardPass(const Array& poseKeyPoints, const cv::Mat& cvInputData); - Array getHands() const; + Array getHandKeyPoints() const; double get(const HandsProperty property) const; @@ -60,9 +60,9 @@ namespace op void checkThread() const; - DELETE_COPY(HandsExtractor); + DELETE_COPY(HandExtractor); }; } } -#endif // OPENPOSE__HANDS__HANDS_EXTRACTOR_HPP +#endif // OPENPOSE__HAND__HAND_EXTRACTOR_HPP diff --git a/include/openpose/experimental/hand/handParameters.hpp b/include/openpose/experimental/hand/handParameters.hpp new file mode 100644 index 00000000..9b6a2a98 --- /dev/null +++ b/include/openpose/experimental/hand/handParameters.hpp @@ -0,0 +1,32 @@ +#ifndef OPENPOSE__HAND__HAND_PARAMETERS_HPP +#define OPENPOSE__HAND__HAND_PARAMETERS_HPP + +#include "enumClasses.hpp" +#include "../../pose/poseParameters.hpp" + +namespace op +{ + const unsigned char HAND_MAX_NUMBER_HANDS = 2; + + const unsigned char HAND_NUMBER_PARTS = 21; + #define HAND_PAIRS_TO_RENDER {0,1, 1,2, 2,3, 3,4, 0,5, 5,6, 6,7, 7,8, 0,9, 9,10, 10,11, 11,12, 0,13, 13,14, 14,15, 15,16, 0,17, 17,18, 18,19, 19,20} + + // Constant Global Parameters + // const unsigned char HAND_MAX_PEOPLE = 1; + + // Constant parameters + const auto HAND_CCN_DECREASE_FACTOR = 8.f; + const unsigned int HAND_MAX_PEAKS = 64u; + const std::string HAND_PROTOTXT{"hand/pose_deploy.prototxt"}; + const std::string HAND_TRAINED_MODEL{"hand/pose_iter_120000.caffemodel"}; + + // Default Model Parameters + // They might be modified on running time + const auto HAND_DEFAULT_NMS_THRESHOLD = 0.1f; + + // Rendering default parameters + const auto HAND_DEFAULT_ALPHA_HANDS = POSE_DEFAULT_ALPHA_POSE; + // const auto HAND_DEFAULT_ALPHA_HEATMAP = POSE_DEFAULT_ALPHA_HEATMAP; +} + +#endif // OPENPOSE__HAND__HAND_PARAMETERS_HPP diff --git a/include/openpose/experimental/hand/handRenderGpu.hpp b/include/openpose/experimental/hand/handRenderGpu.hpp new file mode 100644 index 00000000..9be8412f --- /dev/null +++ b/include/openpose/experimental/hand/handRenderGpu.hpp @@ -0,0 +1,12 @@ +#ifndef OPENPOSE__HAND__GPU_HAND_RENDER_HPP +#define OPENPOSE__HAND__GPU_HAND_RENDER_HPP + +#include +#include "handParameters.hpp" + +namespace op +{ + void renderHandsGpu(float* framePtr, const cv::Size& frameSize, const float* const handsPtr, const int numberHands, const float alphaColorToAdd = HAND_DEFAULT_ALPHA_HANDS); +} + +#endif // OPENPOSE__HAND__GPU_HAND_RENDER_HPP diff --git a/include/openpose/experimental/hands/handsRenderer.hpp b/include/openpose/experimental/hand/handRenderer.hpp similarity index 57% rename from include/openpose/experimental/hands/handsRenderer.hpp rename to include/openpose/experimental/hand/handRenderer.hpp index 4174a332..139513f8 100644 --- a/include/openpose/experimental/hands/handsRenderer.hpp +++ b/include/openpose/experimental/hand/handRenderer.hpp @@ -1,5 +1,5 @@ -#ifndef OPENPOSE__HANDS__HANDS_RENDERER_HPP -#define OPENPOSE__HANDS__HANDS_RENDERER_HPP +#ifndef OPENPOSE__HAND__HAND_RENDERER_HPP +#define OPENPOSE__HAND__HAND_RENDERER_HPP #include #include "../../core/array.hpp" @@ -10,24 +10,24 @@ namespace op { namespace experimental { - class HandsRenderer : public Renderer + class HandRenderer : public Renderer { public: - explicit HandsRenderer(const cv::Size& frameSize); + explicit HandRenderer(const cv::Size& frameSize); - ~HandsRenderer(); + ~HandRenderer(); void initializationOnThread(); - void renderHands(Array& outputData, const Array& hands); + void renderHands(Array& outputData, const Array& handKeyPoints); private: const cv::Size mFrameSize; float* pGpuHands; // GPU aux memory - DELETE_COPY(HandsRenderer); + DELETE_COPY(HandRenderer); }; } } -#endif // OPENPOSE__HANDS__HANDS_RENDERER_HPP +#endif // OPENPOSE__HAND__HAND_RENDERER_HPP diff --git a/include/openpose/experimental/hand/headers.hpp b/include/openpose/experimental/hand/headers.hpp new file mode 100644 index 00000000..74fe189e --- /dev/null +++ b/include/openpose/experimental/hand/headers.hpp @@ -0,0 +1,13 @@ +#ifndef OPENPOSE__HAND__HEADERS_HPP +#define OPENPOSE__HAND__HEADERS_HPP + +// hand module +#include "enumClasses.hpp" +#include "handExtractor.hpp" +#include "handParameters.hpp" +#include "handRenderer.hpp" +#include "handRenderGpu.hpp" +#include "wHandExtractor.hpp" +#include "wHandRenderer.hpp" + +#endif // OPENPOSE__HAND__HEADERS_HPP diff --git a/include/openpose/experimental/hands/wHandsExtractor.hpp b/include/openpose/experimental/hand/wHandExtractor.hpp similarity index 64% rename from include/openpose/experimental/hands/wHandsExtractor.hpp rename to include/openpose/experimental/hand/wHandExtractor.hpp index 5b99c75a..ee6e54e5 100644 --- a/include/openpose/experimental/hands/wHandsExtractor.hpp +++ b/include/openpose/experimental/hand/wHandExtractor.hpp @@ -1,28 +1,28 @@ -#ifndef OPENPOSE__HANDS__W_HANDS_EXTRACTOR_HPP -#define OPENPOSE__HANDS__W_HANDS_EXTRACTOR_HPP +#ifndef OPENPOSE__HAND__W_HAND_EXTRACTOR_HPP +#define OPENPOSE__HAND__W_HAND_EXTRACTOR_HPP #include // std::shared_ptr #include "../../thread/worker.hpp" -#include "handsRenderer.hpp" +#include "handRenderer.hpp" namespace op { namespace experimental { template - class WHandsExtractor : public Worker + class WHandExtractor : public Worker { public: - explicit WHandsExtractor(const std::shared_ptr& handsExtractor); + explicit WHandExtractor(const std::shared_ptr& handExtractor); void initializationOnThread(); void work(TDatums& tDatums); private: - std::shared_ptr spHandsExtractor; + std::shared_ptr spHandExtractor; - DELETE_COPY(WHandsExtractor); + DELETE_COPY(WHandExtractor); }; } } @@ -41,19 +41,19 @@ namespace op namespace experimental { template - WHandsExtractor::WHandsExtractor(const std::shared_ptr& handsExtractor) : - spHandsExtractor{handsExtractor} + WHandExtractor::WHandExtractor(const std::shared_ptr& handExtractor) : + spHandExtractor{handExtractor} { } template - void WHandsExtractor::initializationOnThread() + void WHandExtractor::initializationOnThread() { - spHandsExtractor->initializationOnThread(); + spHandExtractor->initializationOnThread(); } template - void WHandsExtractor::work(TDatums& tDatums) + void WHandExtractor::work(TDatums& tDatums) { try { @@ -66,8 +66,8 @@ namespace op // Extract people hands for (auto& tDatum : *tDatums) { - spHandsExtractor->forwardPass(tDatum.pose, tDatum.cvInputData); - tDatum.hands = spHandsExtractor->getHands(); + spHandExtractor->forwardPass(tDatum.poseKeyPoints, tDatum.cvInputData); + tDatum.handKeyPoints = spHandExtractor->getHandKeyPoints(); } // Profiling speed Profiler::timerEnd(profilerKey); @@ -84,8 +84,8 @@ namespace op } } - COMPILE_TEMPLATE_DATUM(WHandsExtractor); + COMPILE_TEMPLATE_DATUM(WHandExtractor); } } -#endif // OPENPOSE__HANDS__W_HANDS_EXTRACTOR_HPP +#endif // OPENPOSE__HAND__W_HAND_EXTRACTOR_HPP diff --git a/include/openpose/experimental/hands/wHandsRenderer.hpp b/include/openpose/experimental/hand/wHandRenderer.hpp similarity index 66% rename from include/openpose/experimental/hands/wHandsRenderer.hpp rename to include/openpose/experimental/hand/wHandRenderer.hpp index 25ff43cb..7ff7ddc6 100644 --- a/include/openpose/experimental/hands/wHandsRenderer.hpp +++ b/include/openpose/experimental/hand/wHandRenderer.hpp @@ -1,28 +1,28 @@ -#ifndef OPENPOSE__HANDS__W_HANDS_RENDERER_HPP -#define OPENPOSE__HANDS__W_HANDS_RENDERER_HPP +#ifndef OPENPOSE__HAND__W_HAND_RENDERER_HPP +#define OPENPOSE__HAND__W_HAND_RENDERER_HPP #include // std::shared_ptr #include "../../thread/worker.hpp" -#include "handsRenderer.hpp" +#include "handRenderer.hpp" namespace op { namespace experimental { template - class WHandsRenderer : public Worker + class WHandRenderer : public Worker { public: - explicit WHandsRenderer(const std::shared_ptr& handsRenderer); + explicit WHandRenderer(const std::shared_ptr& handRenderer); void initializationOnThread(); void work(TDatums& tDatums); private: - std::shared_ptr spHandsRenderer; + std::shared_ptr spHandRenderer; - DELETE_COPY(WHandsRenderer); + DELETE_COPY(WHandRenderer); }; } } @@ -41,19 +41,19 @@ namespace op namespace experimental { template - WHandsRenderer::WHandsRenderer(const std::shared_ptr& handsRenderer) : - spHandsRenderer{handsRenderer} + WHandRenderer::WHandRenderer(const std::shared_ptr& handRenderer) : + spHandRenderer{handRenderer} { } template - void WHandsRenderer::initializationOnThread() + void WHandRenderer::initializationOnThread() { - spHandsRenderer->initializationOnThread(); + spHandRenderer->initializationOnThread(); } template - void WHandsRenderer::work(TDatums& tDatums) + void WHandRenderer::work(TDatums& tDatums) { try { @@ -65,7 +65,7 @@ namespace op const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); // Render people hands for (auto& tDatum : *tDatums) - spHandsRenderer->renderHands(tDatum.outputData, tDatum.hands); + spHandRenderer->renderHands(tDatum.outputData, tDatum.handKeyPoints); // Profiling speed Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 1000); @@ -81,8 +81,8 @@ namespace op } } - COMPILE_TEMPLATE_DATUM(WHandsRenderer); + COMPILE_TEMPLATE_DATUM(WHandRenderer); } } -#endif // OPENPOSE__HANDS__W_HANDS_RENDERER_HPP +#endif // OPENPOSE__HAND__W_HAND_RENDERER_HPP diff --git a/include/openpose/experimental/hands/enumClasses.hpp b/include/openpose/experimental/hands/enumClasses.hpp deleted file mode 100644 index 953073f8..00000000 --- a/include/openpose/experimental/hands/enumClasses.hpp +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef OPENPOSE__HANDS__ENUM_CLASSES_HPP -#define OPENPOSE__HANDS__ENUM_CLASSES_HPP - -namespace op -{ - enum class HandsProperty : bool - { - NMSThreshold = 0, - Size, - }; -} - -#endif // OPENPOSE__HANDS__ENUM_CLASSES_HPP - diff --git a/include/openpose/experimental/hands/handsParameters.hpp b/include/openpose/experimental/hands/handsParameters.hpp deleted file mode 100644 index 2828ae0e..00000000 --- a/include/openpose/experimental/hands/handsParameters.hpp +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef OPENPOSE__HANDS__HANDS_PARAMETERS_HPP -#define OPENPOSE__HANDS__HANDS_PARAMETERS_HPP - -#include "enumClasses.hpp" -#include "../../pose/poseParameters.hpp" - -namespace op -{ - const unsigned char HANDS_MAX_NUMBER_HANDS = 2; - - const unsigned char HANDS_NUMBER_PARTS = 21; - #define HANDS_PAIRS_TO_RENDER {0,1, 1,2, 2,3, 3,4, 0,5, 5,6, 6,7, 7,8, 0,9, 9,10, 10,11, 11,12, 0,13, 13,14, 14,15, 15,16, 0,17, 17,18, 18,19, 19,20} - - // Constant Global Parameters - // const unsigned char HANDS_MAX_PEOPLE = 1; - - // Constant parameters - const auto HANDS_CCN_DECREASE_FACTOR = 8.f; - const unsigned int HANDS_MAX_PEAKS = 64u; - const std::string HANDS_PROTOTXT{"hand/pose_deploy.prototxt"}; - const std::string HANDS_TRAINED_MODEL{"hand/pose_iter_120000.caffemodel"}; - - // Default Model Parameters - // They might be modified on running time - const auto HANDS_DEFAULT_NMS_THRESHOLD = 0.1f; - - // Rendering default parameters - const auto HANDS_DEFAULT_ALPHA_HANDS = POSE_DEFAULT_ALPHA_POSE; - // const auto HANDS_DEFAULT_ALPHA_HEATMAP = POSE_DEFAULT_ALPHA_HEATMAP; -} - -#endif // OPENPOSE__HANDS__HANDS_PARAMETERS_HPP diff --git a/include/openpose/experimental/hands/handsRenderGpu.hpp b/include/openpose/experimental/hands/handsRenderGpu.hpp deleted file mode 100644 index 7e686456..00000000 --- a/include/openpose/experimental/hands/handsRenderGpu.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef OPENPOSE__HANDS__GPU_HANDS_RENDER_HPP -#define OPENPOSE__HANDS__GPU_HANDS_RENDER_HPP - -#include -#include "handsParameters.hpp" - -namespace op -{ - void renderHandsGpu(float* framePtr, const cv::Size& frameSize, const float* const handsPtr, const int numberHands, const float alphaColorToAdd = HANDS_DEFAULT_ALPHA_HANDS); -} - -#endif // OPENPOSE__HANDS__GPU_HANDS_RENDER_HPP diff --git a/include/openpose/experimental/hands/headers.hpp b/include/openpose/experimental/hands/headers.hpp deleted file mode 100644 index c6a77c10..00000000 --- a/include/openpose/experimental/hands/headers.hpp +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef OPENPOSE__HANDS__HEADERS_HPP -#define OPENPOSE__HANDS__HEADERS_HPP - -// hands module -#include "enumClasses.hpp" -#include "handsExtractor.hpp" -#include "handsParameters.hpp" -#include "handsRenderer.hpp" -#include "handsRenderGpu.hpp" -#include "wHandsExtractor.hpp" -#include "wHandsRenderer.hpp" - -#endif // OPENPOSE__HANDS__HEADERS_HPP diff --git a/include/openpose/experimental/headers.hpp b/include/openpose/experimental/headers.hpp index 6a1f41b0..9cdbc5d8 100644 --- a/include/openpose/experimental/headers.hpp +++ b/include/openpose/experimental/headers.hpp @@ -1,11 +1,14 @@ #ifndef OPENPOSE__EXPERIMENTAL__HEADERS_HPP #define OPENPOSE__EXPERIMENTAL__HEADERS_HPP +// face module +#include "face/headers.hpp" + // filestream module #include "filestream/headers.hpp" // hands module -#include "hands/headers.hpp" +#include "hand/headers.hpp" // producer module // #include "producer/headers.hpp" diff --git a/include/openpose/filestream/poseJsonCocoSaver.hpp b/include/openpose/filestream/poseJsonCocoSaver.hpp index 61868a14..48af8e50 100644 --- a/include/openpose/filestream/poseJsonCocoSaver.hpp +++ b/include/openpose/filestream/poseJsonCocoSaver.hpp @@ -22,7 +22,7 @@ namespace op ~PoseJsonCocoSaver(); - void record(const Array& pose, const int imageId); + void record(const Array& poseKeyPoints, const int imageId); private: JsonOfstream mJsonOfstream; diff --git a/include/openpose/filestream/poseJsonSaver.hpp b/include/openpose/filestream/poseJsonSaver.hpp index e9c86b35..b09d12d2 100644 --- a/include/openpose/filestream/poseJsonSaver.hpp +++ b/include/openpose/filestream/poseJsonSaver.hpp @@ -13,7 +13,7 @@ namespace op public: explicit PoseJsonSaver(const std::string& directoryPath); - void savePoseVector(const std::vector>& poseVector, const std::string& fileName) const; + void savePoseKeyPoints(const std::vector>& poseKeyPointsVector, const std::string& fileName) const; }; } diff --git a/include/openpose/filestream/poseSaver.hpp b/include/openpose/filestream/poseSaver.hpp index 7aaaad28..28181721 100644 --- a/include/openpose/filestream/poseSaver.hpp +++ b/include/openpose/filestream/poseSaver.hpp @@ -12,7 +12,7 @@ namespace op public: PoseSaver(const std::string& directoryPath, const DataFormat format); - void savePoseVector(const std::vector>& poseVector, const std::string& fileName) const; + void savePoseKeyPoints(const std::vector>& poseKeyPointsVector, const std::string& fileName) const; private: const DataFormat mFormat; diff --git a/include/openpose/filestream/wPoseJsonSaver.hpp b/include/openpose/filestream/wPoseJsonSaver.hpp index 03564f0c..0c3e76c8 100644 --- a/include/openpose/filestream/wPoseJsonSaver.hpp +++ b/include/openpose/filestream/wPoseJsonSaver.hpp @@ -61,11 +61,11 @@ namespace op // T* to T auto& tDatumsNoPtr = *tDatums; // Record people pose data in json format - std::vector> poseVector(tDatumsNoPtr.size()); + std::vector> poseKeyPointsVector(tDatumsNoPtr.size()); for (auto i = 0; i < tDatumsNoPtr.size(); i++) - poseVector[i] = tDatumsNoPtr[i].pose; + poseKeyPointsVector[i] = tDatumsNoPtr[i].poseKeyPoints; const auto fileName = (!tDatumsNoPtr[0].name.empty() ? tDatumsNoPtr[0].name : std::to_string(tDatumsNoPtr[0].id)); - spPoseJsonSaver->savePoseVector(poseVector, fileName); + spPoseJsonSaver->savePoseKeyPoints(poseKeyPointsVector, fileName); // Profiling speed Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 1000); diff --git a/include/openpose/filestream/wPoseSaver.hpp b/include/openpose/filestream/wPoseSaver.hpp index bb5d427c..e1e375e0 100644 --- a/include/openpose/filestream/wPoseSaver.hpp +++ b/include/openpose/filestream/wPoseSaver.hpp @@ -62,11 +62,11 @@ namespace op // T* to T auto& tDatumsNoPtr = *tDatums; // Record people pose data - std::vector> poseVector(tDatumsNoPtr.size()); + std::vector> poseKeyPointsVector(tDatumsNoPtr.size()); for (auto i = 0; i < tDatumsNoPtr.size(); i++) - poseVector[i] = tDatumsNoPtr[i].pose; + poseKeyPointsVector[i] = tDatumsNoPtr[i].poseKeyPoints; const auto fileName = (!tDatumsNoPtr[0].name.empty() ? tDatumsNoPtr[0].name : std::to_string(tDatumsNoPtr[0].id)); - spPoseSaver->savePoseVector(poseVector, fileName); + spPoseSaver->savePoseKeyPoints(poseKeyPointsVector, fileName); // Profiling speed Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 1000); diff --git a/include/openpose/gui/guiInfoAdder.hpp b/include/openpose/gui/guiInfoAdder.hpp index b60c2bf7..15df926b 100644 --- a/include/openpose/gui/guiInfoAdder.hpp +++ b/include/openpose/gui/guiInfoAdder.hpp @@ -12,7 +12,7 @@ namespace op public: GuiInfoAdder(const cv::Size& outputSize, const int numberGpus); - void addInfo(cv::Mat& cvOutputData, const Array& pose, const unsigned long long id, const std::string& elementRenderedName); + void addInfo(cv::Mat& cvOutputData, const Array& poseKeyPoints, const unsigned long long id, const std::string& elementRenderedName); private: // Const variables diff --git a/include/openpose/gui/wGuiInfoAdder.hpp b/include/openpose/gui/wGuiInfoAdder.hpp index d3f73305..44b2f2b5 100644 --- a/include/openpose/gui/wGuiInfoAdder.hpp +++ b/include/openpose/gui/wGuiInfoAdder.hpp @@ -58,7 +58,7 @@ namespace op const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); // Add GUI components to frame for (auto& tDatum : *tDatums) - spGuiInfoAdder->addInfo(tDatum.cvOutputData, tDatum.pose, tDatum.id, tDatum.elementRendered.second); + spGuiInfoAdder->addInfo(tDatum.cvOutputData, tDatum.poseKeyPoints, tDatum.id, tDatum.elementRendered.second); // Profiling speed Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 1000); diff --git a/include/openpose/pose/bodyPartConnectorBase.hpp b/include/openpose/pose/bodyPartConnectorBase.hpp index f3acffb1..1296a54b 100644 --- a/include/openpose/pose/bodyPartConnectorBase.hpp +++ b/include/openpose/pose/bodyPartConnectorBase.hpp @@ -7,11 +7,11 @@ namespace op { template - void connectBodyPartsCpu(Array& pose, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, + void connectBodyPartsCpu(Array& poseKeyPoints, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const T interThreshold, const int minSubsetCnt, const T minSubsetScore, const T scaleFactor = 1.f); template - void connectBodyPartsGpu(Array& pose, T* posePtr, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, + void connectBodyPartsGpu(Array& poseKeyPoints, T* posePtr, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const T interThreshold, const int minSubsetCnt, const T minSubsetScore, const T scaleFactor = 1.f); } diff --git a/include/openpose/pose/bodyPartConnectorCaffe.hpp b/include/openpose/pose/bodyPartConnectorCaffe.hpp index f8d43991..d5d0856b 100644 --- a/include/openpose/pose/bodyPartConnectorCaffe.hpp +++ b/include/openpose/pose/bodyPartConnectorCaffe.hpp @@ -36,9 +36,9 @@ namespace op void setScaleNetToOutput(const T scaleNetToOutput); - virtual void Forward_cpu(const std::vector*>& bottom, Array& pose); + virtual void Forward_cpu(const std::vector*>& bottom, Array& poseKeyPoints); - virtual void Forward_gpu(const std::vector*>& bottom, const std::vector*>& top, Array& pose); + virtual void Forward_gpu(const std::vector*>& bottom, const std::vector*>& top, Array& poseKeyPoints); virtual void Backward_cpu(const std::vector*>& top, const std::vector& propagate_down, const std::vector*>& bottom); diff --git a/include/openpose/pose/poseExtractor.hpp b/include/openpose/pose/poseExtractor.hpp index e4d8f27c..1830b907 100644 --- a/include/openpose/pose/poseExtractor.hpp +++ b/include/openpose/pose/poseExtractor.hpp @@ -30,7 +30,7 @@ namespace op virtual const float* getPoseGpuConstPtr() const = 0; - Array getPose() const; + Array getPoseKeyPoints() const; double getScaleNetToOutput() const; @@ -44,7 +44,7 @@ namespace op const PoseModel mPoseModel; const cv::Size mNetOutputSize; const cv::Size mOutputSize; - Array mPose; + Array mPoseKeyPoints; double mScaleNetToOutput; void checkThread() const; diff --git a/include/openpose/pose/poseParameters.hpp b/include/openpose/pose/poseParameters.hpp index 438cd65c..4f592350 100644 --- a/include/openpose/pose/poseParameters.hpp +++ b/include/openpose/pose/poseParameters.hpp @@ -87,6 +87,10 @@ namespace op // Rendering default parameters const auto POSE_DEFAULT_ALPHA_POSE = 0.6f; const auto POSE_DEFAULT_ALPHA_HEATMAP = 0.7f; + + // Auxiliary functions + unsigned char poseBodyPartMapStringToKey(const PoseModel poseModel, const std::string& string); + unsigned char poseBodyPartMapStringToKey(const PoseModel poseModel, const std::vector& strings); } #endif // OPENPOSE__POSE__POSE_PARAMETERS_HPP diff --git a/include/openpose/pose/poseRenderer.hpp b/include/openpose/pose/poseRenderer.hpp index 96ba9eb9..3783e8cf 100644 --- a/include/openpose/pose/poseRenderer.hpp +++ b/include/openpose/pose/poseRenderer.hpp @@ -42,7 +42,7 @@ namespace op void setShowGooglyEyes(const bool showGooglyEyes); - std::pair renderPose(Array& outputData, const Array& pose, const double scaleNetToOutput = -1.); + std::pair renderPose(Array& outputData, const Array& poseKeyPoints, const double scaleNetToOutput = -1.); private: const cv::Size mHeatMapsSize; diff --git a/include/openpose/pose/wPoseExtractor.hpp b/include/openpose/pose/wPoseExtractor.hpp index b0947e12..f32d6b7b 100644 --- a/include/openpose/pose/wPoseExtractor.hpp +++ b/include/openpose/pose/wPoseExtractor.hpp @@ -63,7 +63,7 @@ namespace op { spPoseExtractor->forwardPass(tDatum.inputNetData, tDatum.cvInputData.size()); tDatum.poseHeatMaps = spPoseExtractor->getHeatMaps(); - tDatum.pose = spPoseExtractor->getPose(); + tDatum.poseKeyPoints = spPoseExtractor->getPoseKeyPoints(); tDatum.scaleNetToOutput = spPoseExtractor->getScaleNetToOutput(); } // Profiling speed diff --git a/include/openpose/pose/wPoseRenderer.hpp b/include/openpose/pose/wPoseRenderer.hpp index 11fe480d..f95336e9 100644 --- a/include/openpose/pose/wPoseRenderer.hpp +++ b/include/openpose/pose/wPoseRenderer.hpp @@ -60,7 +60,7 @@ namespace op const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); // Render people pose for (auto& tDatum : *tDatums) - tDatum.elementRendered = spPoseRenderer->renderPose(tDatum.outputData, tDatum.pose, tDatum.scaleNetToOutput); + tDatum.elementRendered = spPoseRenderer->renderPose(tDatum.outputData, tDatum.poseKeyPoints, tDatum.scaleNetToOutput); // Profiling speed Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 1000); diff --git a/include/openpose/utilities/cuda.hu b/include/openpose/utilities/cuda.hu index 61ad0c4f..146e5fdc 100644 --- a/include/openpose/utilities/cuda.hu +++ b/include/openpose/utilities/cuda.hu @@ -140,7 +140,7 @@ namespace op colorR = addWeighted(colorR, colorToAdd[0], alphaColorToAdd); colorG = addWeighted(colorG, colorToAdd[1], alphaColorToAdd); colorB = addWeighted(colorB, colorToAdd[2], alphaColorToAdd); - } + } } #endif // OPENPOSE__UTILITIES_CUDA_HU diff --git a/include/openpose/utilities/render.hu b/include/openpose/utilities/render.hu new file mode 100644 index 00000000..87d18299 --- /dev/null +++ b/include/openpose/utilities/render.hu @@ -0,0 +1,120 @@ +#ifndef OPENPOSE__UTILITIES_RENDER_HU +#define OPENPOSE__UTILITIES_RENDER_HU + +namespace op +{ + inline __device__ void renderKeyPoints(float* targetPtr, float2* sharedMaxs, float2* sharedMins, float* sharedScaleF, + const int globalIdx, const int x, const int y, const int targetWidth, const int targetHeight, + const float* const facePtr, const unsigned char* const partPairsPtr, + const int numberFaces, const int numberParts, const int numberPartPairs, + const float* const rgbColorsPtr, const int numberColors, + const float radius, const float stickwidth, const float threshold, const float alphaColorToAdd) + { + // Fill shared parameters + if (globalIdx < numberFaces) + { + sharedMins[globalIdx].x = targetWidth; + sharedMins[globalIdx].y = targetHeight; + sharedMaxs[globalIdx].x = 0.f; + sharedMaxs[globalIdx].y = 0.f; + for (auto part = 0 ; part < numberParts ; part++) + { + const auto index = 3 * (globalIdx*numberParts + part); + const auto x = facePtr[index]; + const auto y = facePtr[index+1]; + const auto score = facePtr[index+2]; + if (score > threshold) + { + if (x < sharedMins[globalIdx].x) + sharedMins[globalIdx].x = x; + if (x > sharedMaxs[globalIdx].x) + sharedMaxs[globalIdx].x = x; + if (y < sharedMins[globalIdx].y) + sharedMins[globalIdx].y = y; + if (y > sharedMaxs[globalIdx].y) + sharedMaxs[globalIdx].y = y; + } + } + const auto averageX = sharedMaxs[globalIdx].x-sharedMins[globalIdx].x; + const auto averageY = sharedMaxs[globalIdx].y-sharedMins[globalIdx].y; + sharedScaleF[globalIdx] = fastTruncate((averageX + averageY) / 400.f, 0.33f, 1.f); // (averageX + averageY) / 2.f / 400.f + const auto constantToAdd = 50.f; + sharedMaxs[globalIdx].x += constantToAdd; + sharedMaxs[globalIdx].y += constantToAdd; + sharedMins[globalIdx].x -= constantToAdd; + sharedMins[globalIdx].y -= constantToAdd; + } + + __syncthreads(); + + // Fill each (x,y) target pixel + if (x < targetWidth && y < targetHeight) + { + const auto baseIndex = y * targetWidth + x; + auto& b = targetPtr[ baseIndex]; + auto& g = targetPtr[ targetWidth * targetHeight + baseIndex]; + auto& r = targetPtr[2 * targetWidth * targetHeight + baseIndex]; + + for (auto person = 0; person < numberFaces; person++) + { + if (x <= sharedMaxs[person].x && x >= sharedMins[person].x && y <= sharedMaxs[person].y && y >= sharedMins[person].y) + { + // Hand part connections + for (auto facePart = 0; facePart < numberPartPairs; facePart++) + { + const auto bSqrt = sharedScaleF[person] * sharedScaleF[person] * stickwidth * stickwidth; + const auto partA = partPairsPtr[2*facePart]; + const auto partB = partPairsPtr[2*facePart+1]; + const auto indexA = person*numberParts*3 + partA*3; + const auto xA = facePtr[indexA]; + const auto yA = facePtr[indexA + 1]; + const auto scoreA = facePtr[indexA + 2]; + const auto indexB = person*numberParts*3 + partB*3; + const auto xB = facePtr[indexB]; + const auto yB = facePtr[indexB + 1]; + const auto scoreB = facePtr[indexB + 2]; + + if (scoreA > threshold && scoreB > threshold) + { + const auto xP = (xA + xB) / 2.f; + const auto yP = (yA + yB) / 2.f; + const auto aSqrt = (xA - xP) * (xA - xP) + (yA - yP) * (yA - yP); + + const auto angle = atan2f(yB - yA, xB - xA); + const auto sine = sinf(angle); + const auto cosine = cosf(angle); + const auto A = cosine * (x - xP) + sine * (y - yP); + const auto B = sine * (x - xP) - cosine * (y - yP); + + const auto judge = A * A / aSqrt + B * B / bSqrt; + const auto minV = 0.f; + const auto maxV = 1.f; + if (minV <= judge && judge <= maxV) + addColorWeighted(r, g, b, &rgbColorsPtr[(facePart%numberColors)*3], alphaColorToAdd); + } + } + + // Hand part circles + for (unsigned char i = 0; i < numberParts; i++) + { + const auto index = 3 * (person*numberParts + i); + const auto localX = facePtr[index]; + const auto localY = facePtr[index + 1]; + const auto score = facePtr[index + 2]; + + if (score > threshold) + { + const auto dist2 = (x - localX) * (x - localX) + (y - localY) * (y - localY); + const auto minr2 = 0.f; + const auto maxr2 = sharedScaleF[person]*sharedScaleF[person]*radius * radius; + if (minr2 <= dist2 && dist2 <= maxr2) + addColorWeighted(r, g, b, &rgbColorsPtr[(i%numberColors)*3], alphaColorToAdd); + } + } + } + } + } + } +} + +#endif // OPENPOSE__UTILITIES_RENDER_HU diff --git a/include/openpose/wrapper/headers.hpp b/include/openpose/wrapper/headers.hpp index 528a9ab0..80ff7a56 100644 --- a/include/openpose/wrapper/headers.hpp +++ b/include/openpose/wrapper/headers.hpp @@ -3,7 +3,8 @@ // wrapper module #include "wrapper.hpp" -#include "wrapperStructHands.hpp" +#include "wrapperStructFace.hpp" +#include "wrapperStructHand.hpp" #include "wrapperStructInput.hpp" #include "wrapperStructOutput.hpp" #include "wrapperStructPose.hpp" diff --git a/include/openpose/wrapper/wrapper.hpp b/include/openpose/wrapper/wrapper.hpp index 190cf2ec..a2cbfc0d 100644 --- a/include/openpose/wrapper/wrapper.hpp +++ b/include/openpose/wrapper/wrapper.hpp @@ -2,10 +2,11 @@ #define OPENPOSE__WRAPPER__WRAPPER_HPP #include "../thread/headers.hpp" -#include "wrapperStructPose.hpp" -#include "wrapperStructHands.hpp" +#include "wrapperStructFace.hpp" +#include "wrapperStructHand.hpp" #include "wrapperStructInput.hpp" #include "wrapperStructOutput.hpp" +#include "wrapperStructPose.hpp" namespace op { @@ -72,18 +73,40 @@ namespace op */ void setWorkerOutput(const TWorker& worker, const bool workerOnNewThread = true); - // If output is not required, just use this function until the renderOutput argument. Keep the default values for the other parameters - // in order not to display/save any output. + // If output is not required, just use this function until the renderOutput argument. Keep the default values for the other parameters in order not to display/save any output. void configure(const WrapperStructPose& wrapperStructPose, // Producer (set producerSharedPtr = nullptr or use the default WrapperStructInput{} to disable any input) - const WrapperStructInput& wrapperStructInput = WrapperStructInput{}, + const WrapperStructInput& wrapperStructInput, // Consumer (keep default values to disable any output) const WrapperStructOutput& wrapperStructOutput = WrapperStructOutput{}); + // THIS FUNCTION IS NOT IMPLEMENTED YET -> COMING SOON + // Similar to the previos configure, but it includes hand extraction and rendering + void configure(const WrapperStructPose& wrapperStructPose, + // Hand (use the default WrapperStructHand{} to disable any hand detector) + const experimental::WrapperStructHand& wrapperHandStruct, + // Producer (set producerSharedPtr = nullptr or use the default WrapperStructInput{} to disable any input) + const WrapperStructInput& wrapperStructInput, + // Consumer (keep default values to disable any output) + const WrapperStructOutput& wrapperStructOutput = WrapperStructOutput{}); + + // THIS FUNCTION IS NOT IMPLEMENTED YET -> COMING SOON + // Similar to the previos configure, but it includes hand extraction and rendering + void configure(const WrapperStructPose& wrapperStructPose, + // Face (use the default WrapperStructFace{} to disable any face detector) + const experimental::WrapperStructFace& wrapperStructFace, + // Producer (set producerSharedPtr = nullptr or use the default WrapperStructInput{} to disable any input) + const WrapperStructInput& wrapperStructInput, + // Consumer (keep default values to disable any output) + const WrapperStructOutput& wrapperStructOutput = WrapperStructOutput{}); + + // THIS FUNCTION IS NOT IMPLEMENTED YET -> COMING SOON // Similar to the previos configure, but it includes hand extraction and rendering void configure(const WrapperStructPose& wrapperStructPose = WrapperStructPose{}, - // Hand (use the default WrapperStructHands{} to disable any hand detector) - const experimental::WrapperStructHands& wrapperHandStruct = experimental::WrapperStructHands{}, + // Face (use the default WrapperStructFace{} to disable any face detector) + const experimental::WrapperStructFace& wrapperStructFace = experimental::WrapperStructFace{}, + // Hand (use the default WrapperStructHand{} to disable any hand detector) + const experimental::WrapperStructHand& wrapperHandStruct = experimental::WrapperStructHand{}, // Producer (set producerSharedPtr = nullptr or use the default WrapperStructInput{} to disable any input) const WrapperStructInput& wrapperStructInput = WrapperStructInput{}, // Consumer (keep default values to disable any output) @@ -349,7 +372,7 @@ namespace op { try { - configure(wrapperStructPose, experimental::WrapperStructHands{}, wrapperStructInput, wrapperStructOutput); + configure(wrapperStructPose, experimental::WrapperStructFace{}, experimental::WrapperStructHand{}, wrapperStructInput, wrapperStructOutput); } catch (const std::exception& e) { @@ -358,8 +381,37 @@ namespace op } template - void Wrapper::configure(const WrapperStructPose& wrapperStructPose, const experimental::WrapperStructHands& wrapperHandStruct, + void Wrapper::configure(const WrapperStructPose& wrapperStructPose, const experimental::WrapperStructFace& wrapperStructFace, const WrapperStructInput& wrapperStructInput, const WrapperStructOutput& wrapperStructOutput) + { + try + { + configure(wrapperStructPose, wrapperStructFace, experimental::WrapperStructHand{}, wrapperStructInput, wrapperStructOutput); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template + void Wrapper::configure(const WrapperStructPose& wrapperStructPose, const experimental::WrapperStructHand& wrapperHandStruct, + const WrapperStructInput& wrapperStructInput, const WrapperStructOutput& wrapperStructOutput) + { + try + { + configure(wrapperStructPose, experimental::WrapperStructFace{}, wrapperHandStruct, wrapperStructInput, wrapperStructOutput); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template + void Wrapper::configure(const WrapperStructPose& wrapperStructPose, const experimental::WrapperStructFace& wrapperStructFace, + const experimental::WrapperStructHand& wrapperHandStruct, const WrapperStructInput& wrapperStructInput, + const WrapperStructOutput& wrapperStructOutput) { try { @@ -393,8 +445,8 @@ namespace op } if (mUserOutputWs.empty() && mThreadManagerMode != ThreadManagerMode::Asynchronous && mThreadManagerMode != ThreadManagerMode::AsynchronousOut) { - const std::string additionalMessage = " You could also set mThreadManagerMode = mThreadManagerMode::Asynchronous(Out) and/or add your own output worker" - " class before calling this function."; + const std::string additionalMessage = " You could also set mThreadManagerMode = mThreadManagerMode::Asynchronous(Out) and/or add your own" + " output worker class before calling this function."; const auto savingSomething = (!wrapperStructOutput.writeImages.empty() || !wrapperStructOutput.writeVideo.empty() || !wrapperStructOutput.writePose.empty() || !wrapperStructOutput.writePoseJson.empty() || !wrapperStructOutput.writeCocoJson.empty() || !wrapperStructOutput.writeHeatMaps.empty()); @@ -516,15 +568,27 @@ namespace op for (auto i = 0; i < spWPoses.size(); i++) spWPoses.at(i) = {std::make_shared>(poseExtractors.at(i))}; + // Face extractor(s) + if (wrapperStructFace.extractAndRenderFace) + { + for (auto gpuId = 0; gpuId < spWPoses.size(); gpuId++) + { + const auto faceExtractor = std::make_shared( + wrapperStructPose.modelFolder, gpuId + wrapperStructPose.gpuNumberStart, wrapperStructPose.poseModel + ); + spWPoses.at(gpuId).emplace_back(std::make_shared>(faceExtractor)); + } + } + // Hand extractor(s) if (wrapperHandStruct.extractAndRenderHands) { for (auto gpuId = 0; gpuId < spWPoses.size(); gpuId++) { - const auto handsExtractor = std::make_shared( + const auto handExtractor = std::make_shared( wrapperStructPose.modelFolder, gpuId + wrapperStructPose.gpuNumberStart, wrapperStructPose.poseModel ); - spWPoses.at(gpuId).emplace_back(std::make_shared>(handsExtractor)); + spWPoses.at(gpuId).emplace_back(std::make_shared>(handExtractor)); } } @@ -539,15 +603,33 @@ namespace op for (auto i = 0; i < spWPoses.size(); i++) { // Construct hands renderer - const auto handsRenderer = std::make_shared(finalOutputSize); + const auto handRenderer = std::make_shared(finalOutputSize); + // Performance boost -> share spGpuMemoryPtr for all renderers + if (!poseRenderers.empty()) + { + const bool isLastRenderer = (!wrapperStructFace.extractAndRenderFace); + handRenderer->setGpuMemoryAndSetIfLast(poseRenderers.at(i)->getGpuMemoryAndSetAsFirst(), isLastRenderer); + } + // Add worker + spWPoses.at(i).emplace_back(std::make_shared>(handRenderer)); + } + } + + // Face renderer(s) + if (wrapperStructFace.extractAndRenderFace) + { + for (auto i = 0; i < spWPoses.size(); i++) + { + // Construct face renderer + const auto faceRenderer = std::make_shared(finalOutputSize); // Performance boost -> share spGpuMemoryPtr for all renderers if (!poseRenderers.empty()) { const bool isLastRenderer = true; - handsRenderer->setGpuMemoryAndSetIfLast(poseRenderers.at(i)->getGpuMemoryAndSetAsFirst(), isLastRenderer); + faceRenderer->setGpuMemoryAndSetIfLast(poseRenderers.at(i)->getGpuMemoryAndSetAsFirst(), isLastRenderer); } // Add worker - spWPoses.at(i).emplace_back(std::make_shared>(handsRenderer)); + spWPoses.at(i).emplace_back(std::make_shared>(faceRenderer)); } } @@ -567,8 +649,8 @@ namespace op && (wrapperStructPose.poseScaleMode != ScaleMode::InputResolution || (finalOutputSize != producerSize)) && (wrapperStructPose.poseScaleMode != ScaleMode::NetOutputResolution || (finalOutputSize != netOutputSize))) { - auto arrayScaler = std::make_shared(wrapperStructPose.poseScaleMode); - mPostProcessingWs.emplace_back(std::make_shared>(arrayScaler)); + auto keyPointScaler = std::make_shared(wrapperStructPose.poseScaleMode); + mPostProcessingWs.emplace_back(std::make_shared>(keyPointScaler)); } mOutputWs.clear(); diff --git a/include/openpose/wrapper/wrapperStructFace.hpp b/include/openpose/wrapper/wrapperStructFace.hpp new file mode 100644 index 00000000..f2c84712 --- /dev/null +++ b/include/openpose/wrapper/wrapperStructFace.hpp @@ -0,0 +1,32 @@ +#ifndef OPENPOSE__WRAPPER__WRAPPER_STRUCT_FACE_HPP +#define OPENPOSE__WRAPPER__WRAPPER_STRUCT_FACE_HPP + +namespace op +{ + namespace experimental + { + /** + * WrapperStructFace: Face estimation and rendering configuration struct. + * DO NOT USE. CODE TO BE FINISHED. + * WrapperStructFace allows the user to set up the face estimation and rendering parameters that will be used for the OpenPose Wrapper + * class. + */ + struct WrapperStructFace + { + /** + * PROVISIONAL PARAMETER. IT WILL BE CHANGED. + * Whether to extract and render face. + */ + bool extractAndRenderFace; + + /** + * Constructor of the struct. + * It has the recommended and default values we recommend for each element of the struct. + * Since all the elements of the struct are public, they can also be manually filled. + */ + WrapperStructFace(const bool extractAndRenderFace = false); + }; + } +} + +#endif // OPENPOSE__WRAPPER__WRAPPER_STRUCT_FACE_HPP diff --git a/include/openpose/wrapper/wrapperStructHands.hpp b/include/openpose/wrapper/wrapperStructHand.hpp similarity index 55% rename from include/openpose/wrapper/wrapperStructHands.hpp rename to include/openpose/wrapper/wrapperStructHand.hpp index 20a95ed2..998e86e4 100644 --- a/include/openpose/wrapper/wrapperStructHands.hpp +++ b/include/openpose/wrapper/wrapperStructHand.hpp @@ -1,17 +1,17 @@ -#ifndef OPENPOSE__WRAPPER__WRAPPER_STRUCT_HANDS_HPP -#define OPENPOSE__WRAPPER__WRAPPER_STRUCT_HANDS_HPP +#ifndef OPENPOSE__WRAPPER__WRAPPER_STRUCT_HAND_HPP +#define OPENPOSE__WRAPPER__WRAPPER_STRUCT_HAND_HPP namespace op { namespace experimental { /** - * WrapperStructHands: Hands estimation and rendering configuration struct. + * WrapperStructHand: Hands estimation and rendering configuration struct. * DO NOT USE. CODE TO BE FINISHED. - * WrapperStructHands allows the user to set up the hands estimation and rendering parameters that will be used for the OpenPose Wrapper + * WrapperStructHand allows the user to set up the hands estimation and rendering parameters that will be used for the OpenPose Wrapper * class. */ - struct WrapperStructHands + struct WrapperStructHand { /** * PROVISIONAL PARAMETER. IT WILL BE CHANGED. @@ -24,9 +24,9 @@ namespace op * It has the recommended and default values we recommend for each element of the struct. * Since all the elements of the struct are public, they can also be manually filled. */ - WrapperStructHands(const bool extractAndRenderHands = false); + WrapperStructHand(const bool extractAndRenderHands = false); }; } } -#endif // OPENPOSE__WRAPPER__WRAPPER_STRUCT_HANDS_HPP +#endif // OPENPOSE__WRAPPER__WRAPPER_STRUCT_HAND_HPP diff --git a/include/openpose/wrapper/wrapperStructPose.hpp b/include/openpose/wrapper/wrapperStructPose.hpp index 9c389800..9dd7d505 100644 --- a/include/openpose/wrapper/wrapperStructPose.hpp +++ b/include/openpose/wrapper/wrapperStructPose.hpp @@ -25,14 +25,14 @@ namespace op /** * Output size of the final rendered image. * It barely affects performance compared to netInputSize. - * The final Datum.pose can be scaled with respect to outputSize if `poseScaleMode` is set to ScaleMode::OutputResolution, even if the + * The final Datum.poseKeyPoints can be scaled with respect to outputSize if `poseScaleMode` is set to ScaleMode::OutputResolution, even if the * rendering is disabled. */ cv::Size outputSize; /** - * Final scale of the Array Datum.pose and the writen pose data. - * The final Datum.pose can be scaled with respect to input size (ScaleMode::InputResolution), net output size (ScaleMode::NetOutputResolution), + * Final scale of the Array Datum.poseKeyPoints and the writen pose data. + * The final Datum.poseKeyPoints can be scaled with respect to input size (ScaleMode::InputResolution), net output size (ScaleMode::NetOutputResolution), * output rendering size (ScaleMode::OutputResolution), from 0 to 1 (ScaleMode::ZeroToOne), and -1 to 1 (ScaleMode::PlusMinusOne). */ ScaleMode poseScaleMode; diff --git a/src/openpose/core/datum.cpp b/src/openpose/core/datum.cpp index 450537a1..81e968d5 100644 --- a/src/openpose/core/datum.cpp +++ b/src/openpose/core/datum.cpp @@ -18,9 +18,10 @@ namespace op outputData{datum.outputData}, cvOutputData{datum.cvOutputData}, // Resulting Array data - pose{datum.pose}, + poseKeyPoints{datum.poseKeyPoints}, poseHeatMaps{datum.poseHeatMaps}, - hands{datum.hands}, + faceKeyPoints{datum.faceKeyPoints}, + handKeyPoints{datum.handKeyPoints}, // Other parameters scaleInputToOutput{datum.scaleInputToOutput}, scaleNetToOutput{datum.scaleNetToOutput}, @@ -42,9 +43,10 @@ namespace op outputData = datum.outputData; cvOutputData = datum.cvOutputData; // Resulting Array data - pose = datum.pose; + poseKeyPoints = datum.poseKeyPoints; poseHeatMaps = datum.poseHeatMaps, - hands = datum.hands, + faceKeyPoints = datum.faceKeyPoints, + handKeyPoints = datum.handKeyPoints, // Other parameters scaleInputToOutput = datum.scaleInputToOutput; scaleNetToOutput = datum.scaleNetToOutput; @@ -77,9 +79,10 @@ namespace op std::swap(outputData, datum.outputData); std::swap(cvOutputData, datum.cvOutputData); // Resulting Array data - std::swap(pose, datum.pose); + std::swap(poseKeyPoints, datum.poseKeyPoints); std::swap(poseHeatMaps, datum.poseHeatMaps); - std::swap(hands, datum.hands); + std::swap(faceKeyPoints, datum.faceKeyPoints); + std::swap(handKeyPoints, datum.handKeyPoints); // Other parameters std::swap(elementRendered, datum.elementRendered); } @@ -103,9 +106,10 @@ namespace op std::swap(outputData, datum.outputData); std::swap(cvOutputData, datum.cvOutputData); // Resulting Array data - std::swap(pose, datum.pose); + std::swap(poseKeyPoints, datum.poseKeyPoints); std::swap(poseHeatMaps, datum.poseHeatMaps); - std::swap(hands, datum.hands); + std::swap(faceKeyPoints, datum.faceKeyPoints); + std::swap(handKeyPoints, datum.handKeyPoints); // Other parameters scaleInputToOutput = datum.scaleInputToOutput; scaleNetToOutput = datum.scaleNetToOutput; @@ -139,9 +143,10 @@ namespace op datum.outputData = outputData.clone(); datum.cvOutputData = cvOutputData.clone(); // Resulting Array data - datum.pose = pose.clone(); + datum.poseKeyPoints = poseKeyPoints.clone(); datum.poseHeatMaps = poseHeatMaps.clone(); - datum.hands = hands.clone(); + datum.faceKeyPoints = faceKeyPoints.clone(); + datum.handKeyPoints = handKeyPoints.clone(); // Other parameters datum.scaleInputToOutput = scaleInputToOutput; datum.scaleNetToOutput = scaleNetToOutput; diff --git a/src/openpose/core/defineTemplates.cpp b/src/openpose/core/defineTemplates.cpp index 2bd1e2a1..d3decfa8 100644 --- a/src/openpose/core/defineTemplates.cpp +++ b/src/openpose/core/defineTemplates.cpp @@ -2,8 +2,8 @@ namespace op { - DEFINE_TEMPLATE_DATUM(WArrayScaler); DEFINE_TEMPLATE_DATUM(WCvMatToOpInput); DEFINE_TEMPLATE_DATUM(WCvMatToOpOutput); + DEFINE_TEMPLATE_DATUM(WKeyPointScaler); DEFINE_TEMPLATE_DATUM(WOpOutputToCvMat); } diff --git a/src/openpose/core/arrayScaler.cpp b/src/openpose/core/keyPointScaler.cpp similarity index 55% rename from src/openpose/core/arrayScaler.cpp rename to src/openpose/core/keyPointScaler.cpp index 72f094d4..e5664aad 100644 --- a/src/openpose/core/arrayScaler.cpp +++ b/src/openpose/core/keyPointScaler.cpp @@ -1,20 +1,20 @@ -#include "openpose/core/scalePose.hpp" +#include "openpose/core/scaleKeyPoints.hpp" #include "openpose/utilities/errorAndLog.hpp" -#include "openpose/core/arrayScaler.hpp" +#include "openpose/core/keyPointScaler.hpp" namespace op { - ArrayScaler::ArrayScaler(const ScaleMode scalePose) : - mScaleMode{scalePose} + KeyPointScaler::KeyPointScaler(const ScaleMode scaleMode) : + mScaleMode{scaleMode} { } - void ArrayScaler::scale(Array& array, const double scaleInputToOutput, const double scaleNetToOutput, const cv::Size& producerSize) const + void KeyPointScaler::scale(Array& arrayToScale, const double scaleInputToOutput, const double scaleNetToOutput, const cv::Size& producerSize) const { try { - std::vector> arrays{array}; - scale(arrays, scaleInputToOutput, scaleNetToOutput, producerSize); + std::vector> arrayToScalesToScale{arrayToScale}; + scale(arrayToScalesToScale, scaleInputToOutput, scaleNetToOutput, producerSize); } catch (const std::exception& e) { @@ -22,7 +22,7 @@ namespace op } } - void ArrayScaler::scale(std::vector>& arrays, const double scaleInputToOutput, const double scaleNetToOutput, const cv::Size& producerSize) const + void KeyPointScaler::scale(std::vector>& arrayToScalesToScale, const double scaleInputToOutput, const double scaleNetToOutput, const cv::Size& producerSize) const { try { @@ -30,20 +30,20 @@ namespace op { // InputResolution if (mScaleMode == ScaleMode::InputResolution) - for (auto& array : arrays) - scalePose(array, 1./scaleInputToOutput); + for (auto& arrayToScale : arrayToScalesToScale) + scaleKeyPoints(arrayToScale, 1./scaleInputToOutput); // NetOutputResolution else if (mScaleMode == ScaleMode::NetOutputResolution) - for (auto& array : arrays) - scalePose(array, 1./scaleNetToOutput); + for (auto& arrayToScale : arrayToScalesToScale) + scaleKeyPoints(arrayToScale, 1./scaleNetToOutput); // [0,1] else if (mScaleMode == ScaleMode::ZeroToOne) { const auto scale = 1./scaleInputToOutput; const auto scaleX = scale / ((double)producerSize.width - 1.); const auto scaleY = scale / ((double)producerSize.height - 1.); - for (auto& array : arrays) - scalePose(array, scaleX, scaleY); + for (auto& arrayToScale : arrayToScalesToScale) + scaleKeyPoints(arrayToScale, scaleX, scaleY); } // [-1,1] else if (mScaleMode == ScaleMode::PlusMinusOne) @@ -52,8 +52,8 @@ namespace op const auto scaleX = (scale / ((double)producerSize.width - 1.)); const auto scaleY = (scale / ((double)producerSize.height - 1.)); const auto offset = -1.; - for (auto& array : arrays) - scalePose(array, scaleX, scaleY, offset, offset); + for (auto& arrayToScale : arrayToScalesToScale) + scaleKeyPoints(arrayToScale, scaleX, scaleY, offset, offset); } // Unknown else diff --git a/src/openpose/core/resizeAndMergeBase.cu b/src/openpose/core/resizeAndMergeBase.cu index b21e51e6..231a2c43 100644 --- a/src/openpose/core/resizeAndMergeBase.cu +++ b/src/openpose/core/resizeAndMergeBase.cu @@ -82,7 +82,7 @@ namespace op } else { - if (scaleGap <= 0.f) + if (scaleGap <= 0.f && num != targetSize[0]) error("The scale gap must be greater than 0.", __LINE__, __FUNCTION__, __FILE__); const auto sourceNumOffset = channels * sourceChannelOffset; for (auto c = 0; c < channels; c++) diff --git a/src/openpose/core/scalePose.cpp b/src/openpose/core/scaleKeyPoints.cpp similarity index 63% rename from src/openpose/core/scalePose.cpp rename to src/openpose/core/scaleKeyPoints.cpp index acbdcbec..03184bbc 100644 --- a/src/openpose/core/scalePose.cpp +++ b/src/openpose/core/scaleKeyPoints.cpp @@ -1,15 +1,15 @@ #include "openpose/utilities/errorAndLog.hpp" -#include "openpose/core/scalePose.hpp" +#include "openpose/core/scaleKeyPoints.hpp" namespace op { const std::string errorMessage = "This function is only for array of dimension: [sizeA x sizeB x 3]."; - void scalePose(Array& pose, const double scale) + void scaleKeyPoints(Array& keyPoints, const double scale) { try { - scalePose(pose, scale, scale); + scaleKeyPoints(keyPoints, scale, scale); } catch (const std::exception& e) { @@ -17,18 +17,18 @@ namespace op } } - void scalePose(Array& pose, const double scaleX, const double scaleY) + void scaleKeyPoints(Array& keyPoints, const double scaleX, const double scaleY) { try { if (scaleX != 1. && scaleY != 1.) { // Error check - if (!pose.empty() && pose.getSize(2) != 3) + if (!keyPoints.empty() && keyPoints.getSize(2) != 3) error(errorMessage, __LINE__, __FUNCTION__, __FILE__); // Get #people and #parts - const auto numberPeople = pose.getSize(0); - const auto numberParts = pose.getSize(1); + const auto numberPeople = keyPoints.getSize(0); + const auto numberParts = keyPoints.getSize(1); // For each person for (auto person = 0 ; person < numberPeople ; person++) { @@ -36,8 +36,8 @@ namespace op for (auto part = 0 ; part < numberParts ; part++) { const auto finalIndex = 3*(person*numberParts + part); - pose[finalIndex] *= scaleX; - pose[finalIndex+1] *= scaleY; + keyPoints[finalIndex] *= scaleX; + keyPoints[finalIndex+1] *= scaleY; } } } @@ -48,18 +48,18 @@ namespace op } } - void scalePose(Array& pose, const double scaleX, const double scaleY, const double offsetX, const double offsetY) + void scaleKeyPoints(Array& keyPoints, const double scaleX, const double scaleY, const double offsetX, const double offsetY) { try { if (scaleX != 1. && scaleY != 1.) { // Error check - if (!pose.empty() && pose.getSize(2) != 3) + if (!keyPoints.empty() && keyPoints.getSize(2) != 3) error(errorMessage, __LINE__, __FUNCTION__, __FILE__); // Get #people and #parts - const auto numberPeople = pose.getSize(0); - const auto numberParts = pose.getSize(1); + const auto numberPeople = keyPoints.getSize(0); + const auto numberParts = keyPoints.getSize(1); // For each person for (auto person = 0 ; person < numberPeople ; person++) { @@ -67,8 +67,8 @@ namespace op for (auto part = 0 ; part < numberParts ; part++) { const auto finalIndex = 3*(person*numberParts + part); - pose[finalIndex] = pose[finalIndex] * scaleX + offsetX; - pose[finalIndex+1] = pose[finalIndex+1] * scaleY + offsetY; + keyPoints[finalIndex] = keyPoints[finalIndex] * scaleX + offsetX; + keyPoints[finalIndex+1] = keyPoints[finalIndex+1] * scaleY + offsetY; } } } diff --git a/src/openpose/experimental/face/defineTemplates.cpp b/src/openpose/experimental/face/defineTemplates.cpp new file mode 100644 index 00000000..7ef3096b --- /dev/null +++ b/src/openpose/experimental/face/defineTemplates.cpp @@ -0,0 +1,7 @@ +#include "openpose/experimental/face/headers.hpp" + +namespace op +{ + DEFINE_TEMPLATE_DATUM(experimental::WFaceExtractor); + DEFINE_TEMPLATE_DATUM(experimental::WFaceRenderer); +} diff --git a/src/openpose/experimental/face/faceExtractor.cpp b/src/openpose/experimental/face/faceExtractor.cpp new file mode 100644 index 00000000..9ab6cb07 --- /dev/null +++ b/src/openpose/experimental/face/faceExtractor.cpp @@ -0,0 +1,154 @@ +#include // CV_WARP_INVERSE_MAP, CV_INTER_LINEAR +#include "openpose/core/netCaffe.hpp" +#include "openpose/experimental/face/faceParameters.hpp" +#include "openpose/pose/poseParameters.hpp" +#include "openpose/utilities/cuda.hpp" +#include "openpose/utilities/errorAndLog.hpp" +#include "openpose/utilities/fastMath.hpp" +#include "openpose/utilities/openCv.hpp" +#include "openpose/experimental/face/faceExtractor.hpp" +#include "openpose/experimental/face/faceRenderGpu.hpp" // For commented debugging section + +namespace op +{ + namespace experimental + { + FaceExtractor::FaceExtractor(const std::string& modelFolder, const int gpuId, const PoseModel poseModel) : + mNetOutputSize{368, 368}, + mOutputSize{1280, 720}, + mNeck{poseBodyPartMapStringToKey(poseModel, "Neck")}, + mNose{poseBodyPartMapStringToKey(poseModel, std::vector{"Nose", "Head"})}, + mLEar{poseBodyPartMapStringToKey(poseModel, std::vector{"LEar", "Head"})}, + mREar{poseBodyPartMapStringToKey(poseModel, std::vector{"REar", "Head"})}, + mLEye{poseBodyPartMapStringToKey(poseModel, std::vector{"LEye", "Head"})}, + mREye{poseBodyPartMapStringToKey(poseModel, std::vector{"REye", "Head"})}, + spNet{std::make_shared(std::array{1, 3, mNetOutputSize.height, mNetOutputSize.width}, modelFolder + FACE_PROTOTXT, modelFolder + FACE_TRAINED_MODEL, gpuId)}, + spResizeAndMergeCaffe{std::make_shared>()}, + spNmsCaffe{std::make_shared>()}, + mFaceImageCrop{mNetOutputSize.area()*3} + { + try + { +error("Hands and face extraction is not implemented yet. COMING SOON!", __LINE__, __FUNCTION__, __FILE__); + // Properties + for (auto& property : mProperties) + property = 0.; + mProperties[(int)FaceProperty::NMSThreshold] = FACE_DEFAULT_NMS_THRESHOLD; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void FaceExtractor::initializationOnThread() + { + try + { + log("Starting initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + + // Get thread id + mThreadId = {std::this_thread::get_id()}; + + // Caffe net + spNet->initializationOnThread(); + spCaffeNetOutputBlob = ((NetCaffe*)spNet.get())->getOutputBlob(); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + + // HeatMaps extractor blob and layer + spHeatMapsBlob = {std::make_shared>(1,1,1,1)}; + const bool mergeFirstDimension = true; + spResizeAndMergeCaffe->Reshape({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()}, FACE_CCN_DECREASE_FACTOR, mergeFirstDimension); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + + // Pose extractor blob and layer + spPeaksBlob = {std::make_shared>(1,1,1,1)}; + spNmsCaffe->Reshape({spHeatMapsBlob.get()}, {spPeaksBlob.get()}, FACE_MAX_PEAKS, FACE_NUMBER_PARTS); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + + log("Finished initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void FaceExtractor::forwardPass(const Array& poseKeyPoints, const cv::Mat& cvInputData) + { + try + { +UNUSED(poseKeyPoints); +UNUSED(cvInputData); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + Array FaceExtractor::getFaceKeyPoints() const + { + try + { + checkThread(); + return mFaceKeyPoints; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return Array{}; + } + } + + double FaceExtractor::get(const FaceProperty property) const + { + try + { + return mProperties.at((int)property); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return 0.; + } + } + + void FaceExtractor::set(const FaceProperty property, const double value) + { + try + { + mProperties.at((int)property) = {value}; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void FaceExtractor::increase(const FaceProperty property, const double value) + { + try + { + mProperties[(int)property] = mProperties.at((int)property) + value; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void FaceExtractor::checkThread() const + { + try + { + if(mThreadId != std::this_thread::get_id()) + error("The CPU/GPU pointer data cannot be accessed from a different thread.", __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + } +} diff --git a/src/openpose/experimental/face/faceRenderGpu.cu b/src/openpose/experimental/face/faceRenderGpu.cu new file mode 100644 index 00000000..9811ed12 --- /dev/null +++ b/src/openpose/experimental/face/faceRenderGpu.cu @@ -0,0 +1,65 @@ +#include "openpose/experimental/face/faceParameters.hpp" +#include "openpose/utilities/errorAndLog.hpp" +#include "openpose/utilities/cuda.hpp" +#include "openpose/utilities/cuda.hu" +#include "openpose/utilities/render.hu" +#include "openpose/experimental/face/faceRenderGpu.hpp" + +namespace op +{ + const auto THREADS_PER_BLOCK_1D = 32; + + __constant__ const unsigned char PART_PAIRS_GPU[] = FACE_PAIRS_TO_RENDER; + __constant__ const float RGB_COLORS[] = { + 255.f, 255.f, 255.f, + }; + + + + __global__ void renderFaceParts(float* targetPtr, const int targetWidth, const int targetHeight, const float* const facePtr, + const int numberFaces, const float threshold, const float alphaColorToAdd) + { + const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; + const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; + const auto globalIdx = threadIdx.y * blockDim.x + threadIdx.x; + + // Shared parameters + __shared__ float2 sharedMins[FACE_MAX_NUMBER_FACE]; + __shared__ float2 sharedMaxs[FACE_MAX_NUMBER_FACE]; + __shared__ float sharedScaleF[FACE_MAX_NUMBER_FACE]; + + // Other parameters + const auto numberPartPairs = sizeof(PART_PAIRS_GPU) / (2*sizeof(PART_PAIRS_GPU[0])); + const auto numberColors = sizeof(RGB_COLORS) / (3*sizeof(RGB_COLORS[0])); + const auto radius = fastMin(targetWidth, targetHeight) / 120.f; + const auto stickwidth = fastMin(targetWidth, targetHeight) / 250.f; + + // Render key points + renderKeyPoints(targetPtr, sharedMaxs, sharedMins, sharedScaleF, + globalIdx, x, y, targetWidth, targetHeight, facePtr, PART_PAIRS_GPU, numberFaces, + FACE_NUMBER_PARTS, numberPartPairs, RGB_COLORS, numberColors, + radius, stickwidth, threshold, alphaColorToAdd); + } + + void renderFaceGpu(float* framePtr, const cv::Size& frameSize, const float* const facePtr, const int numberFaces, const float alphaColorToAdd) + { + try + { + if (numberFaces > 0) + { + //framePtr = width * height * 3 + //facePtr = 3 (x,y,score) * #Hand parts * # face + const auto threshold = 0.4f; + dim3 threadsPerBlock = dim3{THREADS_PER_BLOCK_1D, THREADS_PER_BLOCK_1D}; + dim3 numBlocks = dim3{getNumberCudaBlocks(frameSize.width, threadsPerBlock.x), getNumberCudaBlocks(frameSize.height, threadsPerBlock.y)}; + // log("numberFaces: " + std::to_string(numberFaces), Priority::Low, __LINE__, __FUNCTION__, __FILE__); + renderFaceParts<<>>(framePtr, frameSize.width, frameSize.height, facePtr, numberFaces, threshold, alphaColorToAdd); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } +} diff --git a/src/openpose/experimental/face/faceRenderer.cpp b/src/openpose/experimental/face/faceRenderer.cpp new file mode 100644 index 00000000..71022038 --- /dev/null +++ b/src/openpose/experimental/face/faceRenderer.cpp @@ -0,0 +1,66 @@ +#include +#include +#include "openpose/experimental/face/faceParameters.hpp" +#include "openpose/experimental/face/faceRenderGpu.hpp" +#include "openpose/utilities/cuda.hpp" +#include "openpose/utilities/errorAndLog.hpp" +#include "openpose/experimental/face/faceRenderer.hpp" + +namespace op +{ + namespace experimental + { + FaceRenderer::FaceRenderer(const cv::Size& frameSize) : + Renderer{(unsigned long long)(frameSize.area() * 3)}, + mFrameSize{frameSize} + { + } + + FaceRenderer::~FaceRenderer() + { + try + { + // Free CUDA pointers - Note that if pointers are 0 (i.e. nullptr), no operation is performed. + cudaFree(pGpuFace); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void FaceRenderer::initializationOnThread() + { + try + { + Renderer::initializationOnThread(); + cudaMalloc((void**)(&pGpuFace), FACE_NUMBER_PARTS * 3 * sizeof(float) ); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void FaceRenderer::renderFace(Array& outputData, const Array& faceKeyPoints) + { + try + { + if (!faceKeyPoints.empty()) + { + cpuToGpuMemoryIfNotCopiedYet(outputData.getPtr()); + cudaMemcpy(pGpuFace, faceKeyPoints.getConstPtr(), FACE_NUMBER_PARTS*3 * sizeof(float), cudaMemcpyHostToDevice); + renderFaceGpu(*spGpuMemoryPtr, mFrameSize, pGpuFace, faceKeyPoints.getSize(0)); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + } + // GPU memory to CPU if last renderer + gpuToCpuMemoryIfLastRenderer(outputData.getPtr()); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + } +} diff --git a/src/openpose/experimental/hand/defineTemplates.cpp b/src/openpose/experimental/hand/defineTemplates.cpp new file mode 100644 index 00000000..23e155b7 --- /dev/null +++ b/src/openpose/experimental/hand/defineTemplates.cpp @@ -0,0 +1,7 @@ +#include "openpose/experimental/hand/headers.hpp" + +namespace op +{ + DEFINE_TEMPLATE_DATUM(experimental::WHandExtractor); + DEFINE_TEMPLATE_DATUM(experimental::WHandRenderer); +} diff --git a/src/openpose/experimental/hand/handExtractor.cpp b/src/openpose/experimental/hand/handExtractor.cpp new file mode 100644 index 00000000..ecd4128d --- /dev/null +++ b/src/openpose/experimental/hand/handExtractor.cpp @@ -0,0 +1,157 @@ +#include // CV_WARP_INVERSE_MAP, CV_INTER_LINEAR +#include "openpose/core/netCaffe.hpp" +#include "openpose/experimental/hand/handParameters.hpp" +#include "openpose/pose/poseParameters.hpp" +#include "openpose/utilities/cuda.hpp" +#include "openpose/utilities/errorAndLog.hpp" +#include "openpose/utilities/fastMath.hpp" +#include "openpose/utilities/openCv.hpp" +#include "openpose/experimental/hand/handExtractor.hpp" +// #include "openpose/experimental/hand/handRenderGpu.hpp" // For commented debugging section + +namespace op +{ + namespace experimental + { + HandExtractor::HandExtractor(const std::string& modelFolder, const int gpuId, const PoseModel poseModel) : + mNetOutputSize{368, 368}, + mOutputSize{1280, 720}, + mRWrist{poseBodyPartMapStringToKey(poseModel, "RWrist")}, + mRElbow{poseBodyPartMapStringToKey(poseModel, "RElbow")}, + mLWrist{poseBodyPartMapStringToKey(poseModel, "LWrist")}, + mLElbow{poseBodyPartMapStringToKey(poseModel, "LElbow")}, + mNeck{poseBodyPartMapStringToKey(poseModel, "Neck")}, + mHeadNose{poseBodyPartMapStringToKey(poseModel, std::vector{"Nose", "Head"})}, + spNet{std::make_shared(std::array{2, 3, mNetOutputSize.height, mNetOutputSize.width}, modelFolder + HAND_PROTOTXT, modelFolder + HAND_TRAINED_MODEL, gpuId)}, + spResizeAndMergeCaffe{std::make_shared>()}, + spNmsCaffe{std::make_shared>()}, + mLeftHandCrop{mNetOutputSize.area()*3}, + mRightHandCrop{mLeftHandCrop.getSize()}, + mScaleLeftHand{100.f}, + mScaleRightHand{mScaleLeftHand} + { + try + { +error("Hands and face extraction is not implemented yet. COMING SOON!", __LINE__, __FUNCTION__, __FILE__); + // Properties + for (auto& property : mProperties) + property = 0.; + mProperties[(int)HandsProperty::NMSThreshold] = HAND_DEFAULT_NMS_THRESHOLD; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void HandExtractor::initializationOnThread() + { + try + { + log("Starting initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + + // Get thread id + mThreadId = {std::this_thread::get_id()}; + + // Caffe net + spNet->initializationOnThread(); + spCaffeNetOutputBlob = ((NetCaffe*)spNet.get())->getOutputBlob(); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + + // HeatMaps extractor blob and layer + spHeatMapsBlob = {std::make_shared>(1,1,1,1)}; + const bool mergeFirstDimension = false; + spResizeAndMergeCaffe->Reshape({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()}, HAND_CCN_DECREASE_FACTOR, mergeFirstDimension); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + + // Pose extractor blob and layer + spPeaksBlob = {std::make_shared>(1,1,1,1)}; + spNmsCaffe->Reshape({spHeatMapsBlob.get()}, {spPeaksBlob.get()}, HAND_MAX_PEAKS, HAND_NUMBER_PARTS); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + + log("Finished initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void HandExtractor::forwardPass(const Array& poseKeyPoints, const cv::Mat& cvInputData) + { + try + { +UNUSED(poseKeyPoints); +UNUSED(cvInputData); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + Array HandExtractor::getHandKeyPoints() const + { + try + { + checkThread(); + return mHands; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return Array{}; + } + } + + double HandExtractor::get(const HandsProperty property) const + { + try + { + return mProperties.at((int)property); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return 0.; + } + } + + void HandExtractor::set(const HandsProperty property, const double value) + { + try + { + mProperties.at((int)property) = {value}; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void HandExtractor::increase(const HandsProperty property, const double value) + { + try + { + mProperties[(int)property] = mProperties.at((int)property) + value; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void HandExtractor::checkThread() const + { + try + { + if(mThreadId != std::this_thread::get_id()) + error("The CPU/GPU pointer data cannot be accessed from a different thread.", __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + } +} diff --git a/src/openpose/experimental/hand/handRenderGpu.cu b/src/openpose/experimental/hand/handRenderGpu.cu new file mode 100644 index 00000000..8c53d6e1 --- /dev/null +++ b/src/openpose/experimental/hand/handRenderGpu.cu @@ -0,0 +1,88 @@ +#include "openpose/experimental/hand/handParameters.hpp" +#include "openpose/utilities/errorAndLog.hpp" +#include "openpose/utilities/cuda.hpp" +#include "openpose/utilities/cuda.hu" +#include "openpose/utilities/render.hu" +#include "openpose/experimental/hand/handRenderGpu.hpp" + +namespace op +{ + const auto THREADS_PER_BLOCK_1D = 32; + + __constant__ const unsigned char PART_PAIRS_GPU[] = HAND_PAIRS_TO_RENDER; + __constant__ const float RGB_COLORS[] = { + 179.f, 0.f, 0.f, + 204.f, 0.f, 0.f, + 230.f, 0.f, 0.f, + 255.f, 0.f, 0.f, + 143.f, 179.f, 0.f, + 163.f, 204.f, 0.f, + 184.f, 230.f, 0.f, + 204.f, 255.f, 0.f, + 0.f, 179.f, 71.f, + 0.f, 204.f, 82.f, + 0.f, 230.f, 92.f, + 0.f, 255.f, 102.f, + 0.f, 71.f, 179.f, + 0.f, 82.f, 204.f, + 0.f, 92.f, 230.f, + 0.f, 102.f, 255.f, + 143.f, 0.f, 179.f, + 163.f, 0.f, 204.f, + 184.f, 0.f, 230.f, + 204.f, 0.f, 255.f, + 179.f, 179.f, 179.f, + 179.f, 179.f, 179.f, + 179.f, 179.f, 179.f, + 179.f, 179.f, 179.f + }; + + + + __global__ void renderHandsParts(float* targetPtr, const int targetWidth, const int targetHeight, const float* const handsPtr, + const int numberHands, const float threshold, const float alphaColorToAdd) + { + const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; + const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; + const auto globalIdx = threadIdx.y * blockDim.x + threadIdx.x; + + // Shared parameters + __shared__ float2 sharedMins[HAND_MAX_NUMBER_HANDS]; + __shared__ float2 sharedMaxs[HAND_MAX_NUMBER_HANDS]; + __shared__ float sharedScaleF[HAND_MAX_NUMBER_HANDS]; + + // Other parameters + const auto numberPartPairs = sizeof(PART_PAIRS_GPU) / (2*sizeof(PART_PAIRS_GPU[0])); + const auto numberColors = sizeof(RGB_COLORS) / (3*sizeof(RGB_COLORS[0])); + const auto radius = fastMin(targetWidth, targetHeight) / 100.f; + const auto stickwidth = fastMin(targetWidth, targetHeight) / 80.f; + + // Render key points + renderKeyPoints(targetPtr, sharedMaxs, sharedMins, sharedScaleF, + globalIdx, x, y, targetWidth, targetHeight, handsPtr, PART_PAIRS_GPU, numberHands, + HAND_NUMBER_PARTS, numberPartPairs, RGB_COLORS, numberColors, + radius, stickwidth, threshold, alphaColorToAdd); + } + + void renderHandsGpu(float* framePtr, const cv::Size& frameSize, const float* const handsPtr, const int numberHands, const float alphaColorToAdd) + { + try + { + if (numberHands > 0) + { + //framePtr = width * height * 3 + //handsPtr = 3 (x,y,score) * #Hand parts * # hands + const auto threshold = 0.05f; + dim3 threadsPerBlock = dim3{THREADS_PER_BLOCK_1D, THREADS_PER_BLOCK_1D}; + dim3 numBlocks = dim3{getNumberCudaBlocks(frameSize.width, threadsPerBlock.x), getNumberCudaBlocks(frameSize.height, threadsPerBlock.y)}; + // log("numberHands: " + std::to_string(numberHands), Priority::Low, __LINE__, __FUNCTION__, __FILE__); + renderHandsParts<<>>(framePtr, frameSize.width, frameSize.height, handsPtr, numberHands, threshold, alphaColorToAdd); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } +} diff --git a/src/openpose/experimental/hand/handRenderer.cpp b/src/openpose/experimental/hand/handRenderer.cpp new file mode 100644 index 00000000..885375fd --- /dev/null +++ b/src/openpose/experimental/hand/handRenderer.cpp @@ -0,0 +1,66 @@ +#include +#include +#include "openpose/experimental/hand/handParameters.hpp" +#include "openpose/experimental/hand/handRenderGpu.hpp" +#include "openpose/utilities/cuda.hpp" +#include "openpose/utilities/errorAndLog.hpp" +#include "openpose/experimental/hand/handRenderer.hpp" + +namespace op +{ + namespace experimental + { + HandRenderer::HandRenderer(const cv::Size& frameSize) : + Renderer{(unsigned long long)(frameSize.area() * 3)}, + mFrameSize{frameSize} + { + } + + HandRenderer::~HandRenderer() + { + try + { + // Free CUDA pointers - Note that if pointers are 0 (i.e. nullptr), no operation is performed. + cudaFree(pGpuHands); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void HandRenderer::initializationOnThread() + { + try + { + Renderer::initializationOnThread(); + cudaMalloc((void**)(&pGpuHands), 2*HAND_NUMBER_PARTS * 3 * sizeof(float) ); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void HandRenderer::renderHands(Array& outputData, const Array& handKeyPoints) + { + try + { + if (!handKeyPoints.empty()) + { + cpuToGpuMemoryIfNotCopiedYet(outputData.getPtr()); + cudaMemcpy(pGpuHands, handKeyPoints.getConstPtr(), 2*HAND_NUMBER_PARTS*3 * sizeof(float), cudaMemcpyHostToDevice); + renderHandsGpu(*spGpuMemoryPtr, mFrameSize, pGpuHands, handKeyPoints.getSize(0)); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + } + // GPU memory to CPU if last renderer + gpuToCpuMemoryIfLastRenderer(outputData.getPtr()); + cudaCheck(__LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + } +} diff --git a/src/openpose/experimental/hands/defineTemplates.cpp b/src/openpose/experimental/hands/defineTemplates.cpp deleted file mode 100644 index 48dbf213..00000000 --- a/src/openpose/experimental/hands/defineTemplates.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "openpose/experimental/hands/headers.hpp" - -namespace op -{ - DEFINE_TEMPLATE_DATUM(experimental::WHandsExtractor); - DEFINE_TEMPLATE_DATUM(experimental::WHandsRenderer); -} diff --git a/src/openpose/experimental/hands/handsExtractor.cpp b/src/openpose/experimental/hands/handsExtractor.cpp deleted file mode 100644 index ca614ae0..00000000 --- a/src/openpose/experimental/hands/handsExtractor.cpp +++ /dev/null @@ -1,117 +0,0 @@ -#include "openpose/utilities/errorAndLog.hpp" -#include "openpose/experimental/hands/handsExtractor.hpp" - -namespace op -{ - namespace experimental - { - HandsExtractor::HandsExtractor(const std::string& modelFolder, const int gpuId, const PoseModel poseModel) : - mNetOutputSize{368, 368}, - mOutputSize{1280, 720}, - mRWrist{0}, - mRElbow{0}, - mLWrist{0}, - mLElbow{0}, - mNeck{0}, - mHeadNose{0}, - spNet{}, - spResizeAndMergeCaffe{}, - spNmsCaffe{}, - mLeftHandCrop{mNetOutputSize.area()*3}, - mRightHandCrop{mLeftHandCrop.getSize()}, - mScaleLeftHand{0.f}, - mScaleRightHand{0.f} - { - UNUSED(modelFolder); - UNUSED(gpuId); - UNUSED(poseModel); - error("Hands code is not ready yet. A first beta version will be included in around 1-2 months. Please, set extractAndRenderHands = false in the OpenPose wrapper.", - __LINE__, __FUNCTION__, __FILE__); - } - - void HandsExtractor::initializationOnThread() - { - error("Hands code is not ready yet. A first beta version will be included in around 1-2 months. Please, set extractAndRenderHands = false in the OpenPose wrapper.", - __LINE__, __FUNCTION__, __FILE__); - } - - void HandsExtractor::forwardPass(const Array& pose, const cv::Mat& cvInputData) - { - try - { - UNUSED(pose); - UNUSED(cvInputData); - error("Hands code is not ready yet. A first beta version will be included in around 1-2 months. Please, set extractAndRenderHands = false in the OpenPose wrapper.", - __LINE__, __FUNCTION__, __FILE__); - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } - - Array HandsExtractor::getHands() const - { - try - { - checkThread(); - return mHands; - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return Array{}; - } - } - - double HandsExtractor::get(const HandsProperty property) const - { - try - { - return mProperties.at((int)property); - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return 0.; - } - } - - void HandsExtractor::set(const HandsProperty property, const double value) - { - try - { - mProperties.at((int)property) = {value}; - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } - - void HandsExtractor::increase(const HandsProperty property, const double value) - { - try - { - mProperties[(int)property] = mProperties.at((int)property) + value; - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } - - void HandsExtractor::checkThread() const - { - try - { - if(mThreadId != std::this_thread::get_id()) - error("The CPU/GPU pointer data cannot be accessed from a different thread.", __LINE__, __FUNCTION__, __FILE__); - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } - } -} diff --git a/src/openpose/experimental/hands/handsRenderGpu.cu b/src/openpose/experimental/hands/handsRenderGpu.cu deleted file mode 100644 index c5e5438e..00000000 --- a/src/openpose/experimental/hands/handsRenderGpu.cu +++ /dev/null @@ -1,18 +0,0 @@ -#include "openpose/utilities/errorAndLog.hpp" -#include "openpose/experimental/hands/handsRenderGpu.hpp" - -namespace op -{ - void renderHandsGpu(float* framePtr, const cv::Size& frameSize, const float* const handsPtr, const int numberHands, const float alphaColorToAdd) - { - try - { - error("Hands code is not ready yet. A first beta version will be included in around 1-2 months. Please, set extractAndRenderHands = false in the OpenPose wrapper.", - __LINE__, __FUNCTION__, __FILE__); - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } -} diff --git a/src/openpose/experimental/hands/handsRenderer.cpp b/src/openpose/experimental/hands/handsRenderer.cpp deleted file mode 100644 index cb09a77b..00000000 --- a/src/openpose/experimental/hands/handsRenderer.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include "openpose/utilities/errorAndLog.hpp" -#include "openpose/experimental/hands/handsRenderer.hpp" - -namespace op -{ - namespace experimental - { - HandsRenderer::HandsRenderer(const cv::Size& frameSize) : - Renderer{(unsigned long long)(frameSize.area() * 3)}, - mFrameSize{frameSize} - { - error("Hands code is not ready yet. A first beta version will be included in around 1-2 months. Please, set extractAndRenderHands = false in the OpenPose wrapper.", - __LINE__, __FUNCTION__, __FILE__); - } - - HandsRenderer::~HandsRenderer() - { - } - - void HandsRenderer::initializationOnThread() - { - try - { - error("Hands code is not ready yet. A first beta version will be included in around 1-2 months. Please, set extractAndRenderHands = false in the OpenPose wrapper.", - __LINE__, __FUNCTION__, __FILE__); - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } - - void HandsRenderer::renderHands(Array& outputData, const Array& hands) - { - try - { - UNUSED(outputData); - UNUSED(hands); - error("Hands code is not ready yet. A first beta version will be included in around 1-2 months. Please, set extractAndRenderHands = false in the OpenPose wrapper.", - __LINE__, __FUNCTION__, __FILE__); - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } - } -} diff --git a/src/openpose/filestream/poseJsonCocoSaver.cpp b/src/openpose/filestream/poseJsonCocoSaver.cpp index 5a5be676..14290547 100644 --- a/src/openpose/filestream/poseJsonCocoSaver.cpp +++ b/src/openpose/filestream/poseJsonCocoSaver.cpp @@ -28,12 +28,12 @@ namespace op } } - void PoseJsonCocoSaver::record(const Array& pose, const int imageId) + void PoseJsonCocoSaver::record(const Array& poseKeyPoints, const int imageId) { try { - const auto numberPeople = pose.getSize(0); - const auto numberBodyParts = pose.getSize(1); + const auto numberPeople = poseKeyPoints.getSize(0); + const auto numberBodyParts = poseKeyPoints.getSize(1); for (auto person = 0 ; person < numberPeople ; person++) { // Comma at any moment but first element @@ -58,16 +58,16 @@ namespace op mJsonOfstream.plainText("1"); mJsonOfstream.comma(); - // keypoints - i.e. pose + // keypoints - i.e. poseKeyPoints mJsonOfstream.key("keypoints"); mJsonOfstream.arrayOpen(); const std::vector indexesInCocoOrder{0, 15, 14, 17, 16, 5, 2, 6, 3, 7, 4, 11, 8, 12, 9, 13, 10}; for (auto bodyPart = 0 ; bodyPart < indexesInCocoOrder.size() ; bodyPart++) { const auto finalIndex = 3*(person*numberBodyParts + indexesInCocoOrder.at(bodyPart)); - mJsonOfstream.plainText(pose[finalIndex]); + mJsonOfstream.plainText(poseKeyPoints[finalIndex]); mJsonOfstream.comma(); - mJsonOfstream.plainText(pose[finalIndex+1]); + mJsonOfstream.plainText(poseKeyPoints[finalIndex+1]); mJsonOfstream.comma(); mJsonOfstream.plainText(1); if (bodyPart < numberBodyParts-1) diff --git a/src/openpose/filestream/poseJsonSaver.cpp b/src/openpose/filestream/poseJsonSaver.cpp index 52333b61..20f022a3 100644 --- a/src/openpose/filestream/poseJsonSaver.cpp +++ b/src/openpose/filestream/poseJsonSaver.cpp @@ -9,21 +9,21 @@ namespace op { } - void PoseJsonSaver::savePoseVector(const std::vector>& poseVector, const std::string& fileName) const + void PoseJsonSaver::savePoseKeyPoints(const std::vector>& poseKeyPointsVector, const std::string& fileName) const { try { // Record json - if (!poseVector.empty()) + if (!poseKeyPointsVector.empty()) { // File path (no extension) const auto fileNameNoExtension = getNextFileName(fileName) + "_pose"; const bool humanReadable = true; - for (auto i = 0; i < poseVector.size(); i++) + for (auto i = 0; i < poseKeyPointsVector.size(); i++) { const auto fileName = fileNameNoExtension + (i != 0 ? "_" + std::to_string(i) : "") + ".json"; - savePoseJson(poseVector[i], fileName, humanReadable); + savePoseJson(poseKeyPointsVector[i], fileName, humanReadable); } } } diff --git a/src/openpose/filestream/poseSaver.cpp b/src/openpose/filestream/poseSaver.cpp index a8cb103c..de8c959b 100644 --- a/src/openpose/filestream/poseSaver.cpp +++ b/src/openpose/filestream/poseSaver.cpp @@ -11,27 +11,27 @@ namespace op { } - void PoseSaver::savePoseVector(const std::vector>& poseVector, const std::string& fileName) const + void PoseSaver::savePoseKeyPoints(const std::vector>& poseKeyPointsVector, const std::string& fileName) const { try { - if (!poseVector.empty()) + if (!poseKeyPointsVector.empty()) { // File path (no extension) const auto fileNameNoExtension = getNextFileName(fileName) + "_pose"; // Get vector of people poses - std::vector cvMatPoses(poseVector.size()); - for (auto i = 0; i < poseVector.size(); i++) - cvMatPoses[i] = poseVector[i].getConstCvMat(); + std::vector cvMatPoses(poseKeyPointsVector.size()); + for (auto i = 0; i < poseKeyPointsVector.size(); i++) + cvMatPoses[i] = poseKeyPointsVector[i].getConstCvMat(); // Get names inside file - std::vector poseVectorNames(cvMatPoses.size()); + std::vector poseKeyPointsVectorNames(cvMatPoses.size()); for (auto i = 0; i < cvMatPoses.size(); i++) - poseVectorNames[i] = {"pose_" + std::to_string(i)}; + poseKeyPointsVectorNames[i] = {"pose_" + std::to_string(i)}; // Record people poses in desired format - saveData(cvMatPoses, poseVectorNames, fileNameNoExtension, mFormat); + saveData(cvMatPoses, poseKeyPointsVectorNames, fileNameNoExtension, mFormat); } } catch (const std::exception& e) diff --git a/src/openpose/gui/guiInfoAdder.cpp b/src/openpose/gui/guiInfoAdder.cpp index 786026cf..2e8ee3cb 100644 --- a/src/openpose/gui/guiInfoAdder.cpp +++ b/src/openpose/gui/guiInfoAdder.cpp @@ -49,7 +49,7 @@ namespace op { } - void GuiInfoAdder::addInfo(cv::Mat& cvOutputData, const Array& pose, const unsigned long long id, const std::string& elementRenderedName) + void GuiInfoAdder::addInfo(cv::Mat& cvOutputData, const Array& poseKeyPoints, const unsigned long long id, const std::string& elementRenderedName) { try { @@ -83,7 +83,7 @@ namespace op // Frame number putTextOnCvMat(cvOutputData, "Frame " + std::to_string(id), {mBorderMargin, (int)(mOutputSize.height - mBorderMargin)}, white, false); // Number people - const auto textToDisplay = std::to_string(pose.getSize(0)) + " people"; + const auto textToDisplay = std::to_string(poseKeyPoints.getSize(0)) + " people"; putTextOnCvMat(cvOutputData, textToDisplay, {(int)(mOutputSize.width - mBorderMargin), (int)(mOutputSize.height - mBorderMargin)}, white, true); } catch (const std::exception& e) diff --git a/src/openpose/pose/bodyPartConnectorBase.cpp b/src/openpose/pose/bodyPartConnectorBase.cpp index 225b9ddc..968ea682 100644 --- a/src/openpose/pose/bodyPartConnectorBase.cpp +++ b/src/openpose/pose/bodyPartConnectorBase.cpp @@ -7,7 +7,7 @@ namespace op { template - void connectBodyPartsCpu(Array& pose, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, + void connectBodyPartsCpu(Array& poseKeyPoints, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const T interThreshold, const int minSubsetCnt, const T minSubsetScore, const T scaleFactor) { try @@ -290,11 +290,11 @@ namespace op error("Bad subsetCounter. Bug in in this function if this happens.", __LINE__, __FUNCTION__, __FILE__); } - // Fill and return pose + // Fill and return poseKeyPoints if (numberPeople > 0) - pose.reset({numberPeople, numberBodyParts, 3}); + poseKeyPoints.reset({numberPeople, numberBodyParts, 3}); else - pose.reset(); + poseKeyPoints.reset(); for (auto person = 0 ; person < validSubsetIndexes.size() ; person++) { const auto& subsetI = subset[validSubsetIndexes[person]].first; @@ -304,15 +304,15 @@ namespace op const auto bodyPartIndex = subsetI[bodyPart]; if (bodyPartIndex > 0) { - pose[baseOffset] = peaksPtr[bodyPartIndex-2] * scaleFactor; - pose[baseOffset + 1] = peaksPtr[bodyPartIndex-1] * scaleFactor; - pose[baseOffset + 2] = peaksPtr[bodyPartIndex]; + poseKeyPoints[baseOffset] = peaksPtr[bodyPartIndex-2] * scaleFactor; + poseKeyPoints[baseOffset + 1] = peaksPtr[bodyPartIndex-1] * scaleFactor; + poseKeyPoints[baseOffset + 2] = peaksPtr[bodyPartIndex]; } else { - pose[baseOffset] = 0.f; - pose[baseOffset + 1] = 0.f; - pose[baseOffset + 2] = 0.f; + poseKeyPoints[baseOffset] = 0.f; + poseKeyPoints[baseOffset + 1] = 0.f; + poseKeyPoints[baseOffset + 2] = 0.f; } } } @@ -323,10 +323,10 @@ namespace op } } - template void connectBodyPartsCpu(Array& pose, const float* const heatMapPtr, const float* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, + template void connectBodyPartsCpu(Array& poseKeyPoints, const float* const heatMapPtr, const float* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const float interThreshold, const int minSubsetCnt, const float minSubsetScore, const float scaleFactor); - template void connectBodyPartsCpu(Array& pose, const double* const heatMapPtr, const double* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, + template void connectBodyPartsCpu(Array& poseKeyPoints, const double* const heatMapPtr, const double* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const double interThreshold, const int minSubsetCnt, const double minSubsetScore, const double scaleFactor); } diff --git a/src/openpose/pose/bodyPartConnectorBase.cu b/src/openpose/pose/bodyPartConnectorBase.cu index d922f0a2..d3610693 100644 --- a/src/openpose/pose/bodyPartConnectorBase.cu +++ b/src/openpose/pose/bodyPartConnectorBase.cu @@ -5,12 +5,12 @@ namespace op { template - void connectBodyPartsGpu(Array& pose, T* posePtr, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, + void connectBodyPartsGpu(Array& poseKeyPoints, T* posePtr, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const T interThreshold, const int minSubsetCnt, const T minSubsetScore, const T scaleFactor) { try { - UNUSED(pose); + UNUSED(poseKeyPoints); UNUSED(posePtr); UNUSED(heatMapPtr); UNUSED(peaksPtr); @@ -31,10 +31,10 @@ namespace op } } - template void connectBodyPartsGpu(Array& pose, float* posePtr, const float* const heatMapPtr, const float* const peaksPtr, const PoseModel poseModel, + template void connectBodyPartsGpu(Array& poseKeyPoints, float* posePtr, const float* const heatMapPtr, const float* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const float interThreshold, const int minSubsetCnt, const float minSubsetScore, const float scaleFactor); - template void connectBodyPartsGpu(Array& pose, double* posePtr, const double* const heatMapPtr, const double* const peaksPtr, const PoseModel poseModel, + template void connectBodyPartsGpu(Array& poseKeyPoints, double* posePtr, const double* const heatMapPtr, const double* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const double interThreshold, const int minSubsetCnt, const double minSubsetScore, const double scaleFactor); } diff --git a/src/openpose/pose/bodyPartConnectorCaffe.cpp b/src/openpose/pose/bodyPartConnectorCaffe.cpp index be3e158f..f420a1a3 100644 --- a/src/openpose/pose/bodyPartConnectorCaffe.cpp +++ b/src/openpose/pose/bodyPartConnectorCaffe.cpp @@ -131,7 +131,7 @@ namespace op } template - void BodyPartConnectorCaffe::Forward_cpu(const std::vector*>& bottom, Array& pose) + void BodyPartConnectorCaffe::Forward_cpu(const std::vector*>& bottom, Array& poseKeyPoints) { try { @@ -139,7 +139,7 @@ namespace op const auto* const heatMapsPtr = heatMapsBlob->cpu_data(); // ~8.5ms / 114 const auto* const peaksPtr = bottom.at(1)->cpu_data(); // ~0.02ms const auto maxPeaks = mTopSize[1]; - connectBodyPartsCpu(pose, heatMapsPtr, peaksPtr, mPoseModel, cv::Size{heatMapsBlob->shape(3), heatMapsBlob->shape(2)}, maxPeaks, + connectBodyPartsCpu(poseKeyPoints, heatMapsPtr, peaksPtr, mPoseModel, cv::Size{heatMapsBlob->shape(3), heatMapsBlob->shape(2)}, maxPeaks, mInterMinAboveThreshold, mInterThreshold, mMinSubsetCnt, mMinSubsetScore, mScaleNetToOutput); } catch (const std::exception& e) @@ -149,7 +149,7 @@ namespace op } template - void BodyPartConnectorCaffe::Forward_gpu(const std::vector*>& bottom, const std::vector*>& top, Array& pose) + void BodyPartConnectorCaffe::Forward_gpu(const std::vector*>& bottom, const std::vector*>& top, Array& poseKeyPoints) { try { @@ -157,7 +157,7 @@ namespace op const auto* const heatMapsPtr = heatMapsBlob->gpu_data(); const auto* const peaksPtr = bottom.at(1)->gpu_data(); const auto maxPeaks = mTopSize[1]; - connectBodyPartsGpu(pose, top.at(0)->mutable_gpu_data(), heatMapsPtr, peaksPtr, mPoseModel, cv::Size{heatMapsBlob->shape(3), heatMapsBlob->shape(2)}, maxPeaks, + connectBodyPartsGpu(poseKeyPoints, top.at(0)->mutable_gpu_data(), heatMapsPtr, peaksPtr, mPoseModel, cv::Size{heatMapsBlob->shape(3), heatMapsBlob->shape(2)}, maxPeaks, mInterMinAboveThreshold, mInterThreshold, mMinSubsetCnt, mMinSubsetScore, mScaleNetToOutput); } catch (const std::exception& e) diff --git a/src/openpose/pose/poseExtractor.cpp b/src/openpose/pose/poseExtractor.cpp index be34ec6b..6765abb6 100644 --- a/src/openpose/pose/poseExtractor.cpp +++ b/src/openpose/pose/poseExtractor.cpp @@ -163,12 +163,12 @@ namespace op } } - Array PoseExtractor::getPose() const + Array PoseExtractor::getPoseKeyPoints() const { try { checkThread(); - return mPose; + return mPoseKeyPoints; } catch (const std::exception& e) { diff --git a/src/openpose/pose/poseExtractorCaffe.cpp b/src/openpose/pose/poseExtractorCaffe.cpp index d702b750..a9bf9b46 100644 --- a/src/openpose/pose/poseExtractorCaffe.cpp +++ b/src/openpose/pose/poseExtractorCaffe.cpp @@ -107,8 +107,8 @@ namespace op spBodyPartConnectorCaffe->setMinSubsetScore((float)get(PoseProperty::ConnectMinSubsetScore)); // GPU version not implemented yet - spBodyPartConnectorCaffe->Forward_cpu({spHeatMapsBlob.get(), spPeaksBlob.get()}, mPose); - // spBodyPartConnectorCaffe->Forward_gpu({spHeatMapsBlob.get(), spPeaksBlob.get()}, {spPoseBlob.get()}, mPose); + spBodyPartConnectorCaffe->Forward_cpu({spHeatMapsBlob.get(), spPeaksBlob.get()}, mPoseKeyPoints); + // spBodyPartConnectorCaffe->Forward_gpu({spHeatMapsBlob.get(), spPeaksBlob.get()}, {spPoseBlob.get()}, mPoseKeyPoints); } catch (const std::exception& e) { diff --git a/src/openpose/pose/poseParameters.cpp b/src/openpose/pose/poseParameters.cpp new file mode 100644 index 00000000..f89bd8ef --- /dev/null +++ b/src/openpose/pose/poseParameters.cpp @@ -0,0 +1,37 @@ +#include "openpose/utilities/errorAndLog.hpp" +#include "openpose/pose/poseParameters.hpp" + +namespace op +{ + unsigned char poseBodyPartMapStringToKey(const PoseModel poseModel, const std::vector& strings) + { + try + { + const auto& poseBodyPartMapping = POSE_BODY_PART_MAPPING[(int)poseModel]; + for (auto& string : strings) + for (auto& pair : poseBodyPartMapping) + if (pair.second == string) + return pair.first; + error("String(s) could not be found.", __LINE__, __FUNCTION__, __FILE__); + return 0; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return 0; + } + } + + unsigned char poseBodyPartMapStringToKey(const PoseModel poseModel, const std::string& string) + { + try + { + return poseBodyPartMapStringToKey(poseModel, std::vector{string}); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return 0; + } + } +} diff --git a/src/openpose/pose/poseRenderGpu.cu b/src/openpose/pose/poseRenderGpu.cu index 468e58b4..8a62e0f5 100644 --- a/src/openpose/pose/poseRenderGpu.cu +++ b/src/openpose/pose/poseRenderGpu.cu @@ -3,6 +3,7 @@ #include "openpose/utilities/errorAndLog.hpp" #include "openpose/utilities/cuda.hpp" #include "openpose/utilities/cuda.hu" +#include "openpose/utilities/render.hu" #include "openpose/pose/poseRenderGpu.hpp" namespace op @@ -141,162 +142,29 @@ namespace op colorPtr.z *= rad; } - __global__ void renderPoseCoco(float* targetPtr, const int targetWidth, const int targetHeight, const float* const posePtr, - const int numberPeople, const float threshold, const bool googlyEyes, const float blendOriginalFrame, const float alphaColorToAdd) + __global__ void renderPoseCoco(float* targetPtr, const int targetWidth, const int targetHeight, const float* const posePtr, const int numberPeople, + const float threshold, const bool googlyEyes, const float blendOriginalFrame, const float alphaColorToAdd) { - //posePtr has length 3 * 18 * numberPeople const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; const auto globalIdx = threadIdx.y * blockDim.x + threadIdx.x; - __shared__ float sharedPoses[POSE_COCO_NUMBER_PARTS*3*POSE_MAX_PEOPLE]; + // Shared parameters __shared__ float2 sharedMins[POSE_MAX_PEOPLE]; __shared__ float2 sharedMaxs[POSE_MAX_PEOPLE]; __shared__ float sharedScaleF[POSE_MAX_PEOPLE]; - if (globalIdx < numberPeople) - { - sharedMins[globalIdx].x = targetWidth; - sharedMins[globalIdx].y = targetHeight; - sharedMaxs[globalIdx].x = 0.f; - sharedMaxs[globalIdx].y = 0.f; - for (auto part = 0 ; part < POSE_COCO_NUMBER_PARTS ; part++) - { - const auto index = 3 * (globalIdx*POSE_COCO_NUMBER_PARTS + part); - const auto x = posePtr[index]; - const auto y = posePtr[index+1]; - const auto score = posePtr[index+2]; - sharedPoses[index] = x; - sharedPoses[index+1] = y; - sharedPoses[index+2] = score; - if (score > threshold) - { - if (x < sharedMins[globalIdx].x) - sharedMins[globalIdx].x = x; - if (x > sharedMaxs[globalIdx].x) - sharedMaxs[globalIdx].x = x; - if (y < sharedMins[globalIdx].y) - sharedMins[globalIdx].y = y; - if (y > sharedMaxs[globalIdx].y) - sharedMaxs[globalIdx].y = y; - } - } - const auto averageX = sharedMaxs[globalIdx].x-sharedMins[globalIdx].x; - const auto averageY = sharedMaxs[globalIdx].y-sharedMins[globalIdx].y; - sharedScaleF[globalIdx] = fastTruncate((averageX + averageY) / 400.f, 0.33f, 1.f); // (averageX + averageY) / 2.f / 400.f - const auto constantToAdd = 50.f; - sharedMaxs[globalIdx].x += constantToAdd; - sharedMaxs[globalIdx].y += constantToAdd; - sharedMins[globalIdx].x -= constantToAdd; - sharedMins[globalIdx].y -= constantToAdd; - } - __syncthreads(); + // Other parameters + const auto numberPartPairs = sizeof(COCO_PAIRS_GPU) / (2*sizeof(COCO_PAIRS_GPU[0])); + const auto numberColors = sizeof(COCO_RGB_COLORS) / (3*sizeof(COCO_RGB_COLORS[0])); + const auto radius = fastMin(targetWidth, targetHeight) / 100.f; + const auto stickwidth = fastMin(targetWidth, targetHeight) / 120.f; - const auto numberBodyParts = sizeof(COCO_PAIRS_GPU)/(2*sizeof(COCO_PAIRS_GPU[0])); - const auto numberColors = sizeof(COCO_RGB_COLORS)/(3*sizeof(COCO_RGB_COLORS[0])); - const auto radius = targetHeight / 100.f; - const auto stickwidth = targetHeight / 120.f; - - if (x < targetWidth && y < targetHeight) - { - const auto baseIndex = y * targetWidth + x; - auto& b = targetPtr[ baseIndex]; - auto& g = targetPtr[ targetWidth * targetHeight + baseIndex]; - auto& r = targetPtr[2 * targetWidth * targetHeight + baseIndex]; - if (!blendOriginalFrame) - { - b = 0.f; - g = 0.f; - r = 0.f; - } - - for (auto person = 0; person < numberPeople; person++) - { - if (x <= sharedMaxs[person].x && x >= sharedMins[person].x && y <= sharedMaxs[person].y && y >= sharedMins[person].y) - { - // Body part connections - for (auto bodyPart = 0; bodyPart < numberBodyParts; bodyPart++) - { - const auto bSqrt = sharedScaleF[person] * sharedScaleF[person] * stickwidth * stickwidth; - const auto partA = COCO_PAIRS_GPU[2*bodyPart]; - const auto partB = COCO_PAIRS_GPU[2*bodyPart+1]; - const auto indexA = person*POSE_COCO_NUMBER_PARTS*3 + partA*3; - const auto xA = sharedPoses[indexA]; - const auto yA = sharedPoses[indexA + 1]; - const auto valueA = sharedPoses[indexA + 2]; - const auto indexB = person*POSE_COCO_NUMBER_PARTS*3 + partB*3; - const auto xB = sharedPoses[indexB]; - const auto yB = sharedPoses[indexB + 1]; - const auto valueB = sharedPoses[indexB + 2]; - - if (valueA > threshold && valueB > threshold) - { - const auto xP = (xA + xB) / 2.f; - const auto yP = (yA + yB) / 2.f; - const auto aSqrt = (xA - xP) * (xA - xP) + (yA - yP) * (yA - yP); - - const auto angle = atan2f(yB - yA, xB - xA); - const auto sine = sinf(angle); - const auto cosine = cosf(angle); - const auto A = cosine * (x - xP) + sine * (y - yP); - const auto B = sine * (x - xP) - cosine * (y - yP); - - const auto judge = A * A / aSqrt + B * B / bSqrt; - const auto minV = 0.f; - const auto maxV = 1.f; - if (minV <= judge && judge <= maxV) - { - const auto index = (bodyPart%numberColors)*3; - addColorWeighted(r, g, b, &COCO_RGB_COLORS[index], alphaColorToAdd); - } - } - } - - // Body part circles - for (unsigned char i = 0; i < POSE_COCO_NUMBER_PARTS; i++) - { - const auto index = 3 * (person*POSE_COCO_NUMBER_PARTS + i); - const auto localX = sharedPoses[index]; - const auto localY = sharedPoses[index + 1]; - const auto value = sharedPoses[index + 2]; - - if (value > threshold) - { - const auto dist2 = (x - localX) * (x - localX) + (y - localY) * (y - localY); - - if (googlyEyes && (i==14 || i==15)) - { - const auto minr2 = sharedScaleF[person]*sharedScaleF[person]*(2.5*radius-2)*(2.5*radius-2); - const auto maxr2 = sharedScaleF[person]*sharedScaleF[person]*2.5*2.5*radius*radius; - if (dist2 <= maxr2) - { - float colorToAdd [3] = {0., 0., 0.}; - if (dist2 <= minr2) - for (auto& color : colorToAdd) - color = {255.f}; - if (dist2 <= minr2*0.6) - { - const auto dist3 = (x-4 - localX) * (x-4 - localX) + (y - localY+4) * (y - localY+4); - if (dist3 > 3.75f*3.75f) - for (auto& color : colorToAdd) - color = {0.f}; - } - const auto alphaColorToAdd = 0.9f; - addColorWeighted(r, g, b, colorToAdd, alphaColorToAdd); - } - } - else - { - const auto minr2 = 0.f; - const auto maxr2 = sharedScaleF[person]*sharedScaleF[person]*radius * radius; - if (minr2 <= dist2 && dist2 <= maxr2) - addColorWeighted(r, g, b, &COCO_RGB_COLORS[(i%numberColors)*3], alphaColorToAdd); - } - } - } - } - } - } + // Render key points + renderKeyPoints(targetPtr, sharedMaxs, sharedMins, sharedScaleF, + globalIdx, x, y, targetWidth, targetHeight, posePtr, COCO_PAIRS_GPU, numberPeople, + POSE_COCO_NUMBER_PARTS, numberPartPairs, COCO_RGB_COLORS, numberColors, + radius, stickwidth, threshold, alphaColorToAdd); } __global__ void renderPoseMpi29Parts(float* targetPtr, const int targetWidth, const int targetHeight, const float* const posePtr, @@ -305,25 +173,13 @@ namespace op //posePtr has length 3 * 15 * numberPeople const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; - const auto globalIdx = threadIdx.y * blockDim.x + threadIdx.x; - - __shared__ float sharedPoses[POSE_MPI_NUMBER_PARTS*3*POSE_MAX_PEOPLE]; - if (globalIdx < numberPeople * POSE_MPI_NUMBER_PARTS) - { - const auto index = 3*globalIdx; - sharedPoses[index] = posePtr[index]; - sharedPoses[index+1] = posePtr[index+1]; - sharedPoses[index+2] = posePtr[index+2]; - } - - __syncthreads(); - - const auto numberBodyParts = sizeof(MPI_PAIRS_GPU)/(2*sizeof(MPI_PAIRS_GPU[0])); - const auto radius = 3.f*targetHeight / 200.0f; - const auto stickwidth = targetHeight / 60.0f; if (x < targetWidth && y < targetHeight) { + const auto numberBodyParts = sizeof(MPI_PAIRS_GPU)/(2*sizeof(MPI_PAIRS_GPU[0])); + const auto radius = 3.f*targetHeight / 200.0f; + const auto stickwidth = targetHeight / 60.0f; + const auto blueIndex = y * targetWidth + x; auto& b = targetPtr[ blueIndex]; auto& g = targetPtr[ targetWidth * targetHeight + blueIndex]; @@ -343,12 +199,12 @@ namespace op auto bSqrt = stickwidth * stickwidth; //fixed const auto partA = MPI_PAIRS_GPU[2*bodyPart]; const auto partB = MPI_PAIRS_GPU[2*bodyPart+1]; - const auto xA = sharedPoses[person*POSE_MPI_NUMBER_PARTS*3 + partA*3]; - const auto yA = sharedPoses[person*POSE_MPI_NUMBER_PARTS*3 + partA*3 + 1]; - const auto valueA = sharedPoses[person*POSE_MPI_NUMBER_PARTS*3 + partA*3 + 2]; - const auto xB = sharedPoses[person*POSE_MPI_NUMBER_PARTS*3 + partB*3]; - const auto yB = sharedPoses[person*POSE_MPI_NUMBER_PARTS*3 + partB*3 + 1]; - const auto valueB = sharedPoses[person*POSE_MPI_NUMBER_PARTS*3 + partB*3 + 2]; + const auto xA = posePtr[person*POSE_MPI_NUMBER_PARTS*3 + partA*3]; + const auto yA = posePtr[person*POSE_MPI_NUMBER_PARTS*3 + partA*3 + 1]; + const auto valueA = posePtr[person*POSE_MPI_NUMBER_PARTS*3 + partA*3 + 2]; + const auto xB = posePtr[person*POSE_MPI_NUMBER_PARTS*3 + partB*3]; + const auto yB = posePtr[person*POSE_MPI_NUMBER_PARTS*3 + partB*3 + 1]; + const auto valueB = posePtr[person*POSE_MPI_NUMBER_PARTS*3 + partB*3 + 2]; if (valueA > threshold && valueB > threshold) { const auto xP = (xA + xB) / 2.f; @@ -379,9 +235,9 @@ namespace op for (unsigned char i = 0; i < POSE_MPI_NUMBER_PARTS; i++) //for every point { const auto index = 3 * (person*POSE_MPI_NUMBER_PARTS + i); - const auto localX = sharedPoses[index]; - const auto localY = sharedPoses[index + 1]; - const auto value = sharedPoses[index + 2]; + const auto localX = posePtr[index]; + const auto localY = posePtr[index + 1]; + const auto value = posePtr[index + 2]; if (value > threshold) if ((x - localX) * (x - localX) + (y - localY) * (y - localY) <= radius * radius) @@ -548,7 +404,7 @@ namespace op } } - inline void gpuRenderPartAffinityAux(float* framePtr, const PoseModel poseModel, const cv::Size& frameSize, const float* const heatMapPtr, + inline void renderKeyPointsPartAffinityAux(float* framePtr, const PoseModel poseModel, const cv::Size& frameSize, const float* const heatMapPtr, const cv::Size& heatMapSize, const float scaleToKeepRatio, const int part, const int partsToRender, const float alphaBlending) { try @@ -656,7 +512,7 @@ namespace op { try { - gpuRenderPartAffinityAux(framePtr, poseModel, frameSize, heatMapPtr, heatMapSize, scaleToKeepRatio, part, 1, alphaBlending); + renderKeyPointsPartAffinityAux(framePtr, poseModel, frameSize, heatMapPtr, heatMapSize, scaleToKeepRatio, part, 1, alphaBlending); } catch (const std::exception& e) { @@ -670,7 +526,7 @@ namespace op try { const auto numberBodyPartPairs = POSE_BODY_PART_PAIRS[(int)poseModel].size()/2; - gpuRenderPartAffinityAux(framePtr, poseModel, frameSize, heatMapPtr, heatMapSize, scaleToKeepRatio, POSE_NUMBER_BODY_PARTS[(int)poseModel]+1, + renderKeyPointsPartAffinityAux(framePtr, poseModel, frameSize, heatMapPtr, heatMapSize, scaleToKeepRatio, POSE_NUMBER_BODY_PARTS[(int)poseModel]+1, numberBodyPartPairs, alphaBlending); } catch (const std::exception& e) diff --git a/src/openpose/pose/poseRenderer.cpp b/src/openpose/pose/poseRenderer.cpp index f39d5c8a..418102ac 100644 --- a/src/openpose/pose/poseRenderer.cpp +++ b/src/openpose/pose/poseRenderer.cpp @@ -211,7 +211,7 @@ namespace op } } - std::pair PoseRenderer::renderPose(Array& outputData, const Array& pose, const double scaleNetToOutput) + std::pair PoseRenderer::renderPose(Array& outputData, const Array& poseKeyPoints, const double scaleNetToOutput) { try { @@ -221,7 +221,7 @@ namespace op const auto elementRendered = mElementToRender.load(); // I prefer std::round(T&) over intRound(T) for std::atomic std::string elementRenderedName; - const auto numberPeople = pose.getSize(0); + const auto numberPeople = poseKeyPoints.getSize(0); // GPU rendering if (numberPeople > 0 || elementRendered != 0 || !mBlendOriginalFrame) @@ -230,11 +230,11 @@ namespace op cudaCheck(__LINE__, __FUNCTION__, __FILE__); const auto numberBodyParts = POSE_NUMBER_BODY_PARTS[(int)mPoseModel]; const auto numberBodyPartsPlusBkg = numberBodyParts+1; - // Draw pose + // Draw poseKeyPoints if (elementRendered == 0) { - if (!pose.empty()) - cudaMemcpy(pGpuPose, pose.getConstPtr(), numberPeople * numberBodyParts * 3 * sizeof(float), cudaMemcpyHostToDevice); + if (!poseKeyPoints.empty()) + cudaMemcpy(pGpuPose, poseKeyPoints.getConstPtr(), numberPeople * numberBodyParts * 3 * sizeof(float), cudaMemcpyHostToDevice); renderPoseGpu(*spGpuMemoryPtr, mPoseModel, numberPeople, mOutputSize, pGpuPose, mShowGooglyEyes, mBlendOriginalFrame, mAlphaPose); } else diff --git a/src/openpose/wrapper/wrapperStructFace.cpp b/src/openpose/wrapper/wrapperStructFace.cpp new file mode 100644 index 00000000..a5b5e7e8 --- /dev/null +++ b/src/openpose/wrapper/wrapperStructFace.cpp @@ -0,0 +1,12 @@ +#include "openpose/wrapper/wrapperStructFace.hpp" + +namespace op +{ + namespace experimental + { + WrapperStructFace::WrapperStructFace(const bool extractAndRenderFace_) : + extractAndRenderFace{extractAndRenderFace_} + { + } + } +} diff --git a/src/openpose/wrapper/wrapperStructHands.cpp b/src/openpose/wrapper/wrapperStructHand.cpp similarity index 50% rename from src/openpose/wrapper/wrapperStructHands.cpp rename to src/openpose/wrapper/wrapperStructHand.cpp index 36f2d779..e2f2dc46 100644 --- a/src/openpose/wrapper/wrapperStructHands.cpp +++ b/src/openpose/wrapper/wrapperStructHand.cpp @@ -1,10 +1,10 @@ -#include "openpose/wrapper/wrapperStructHands.hpp" +#include "openpose/wrapper/wrapperStructHand.hpp" namespace op { namespace experimental { - WrapperStructHands::WrapperStructHands(const bool extractAndRenderHands_) : + WrapperStructHand::WrapperStructHand(const bool extractAndRenderHands_) : extractAndRenderHands{extractAndRenderHands_} { }