vhad/openpose
1
0
Fork 0
mirror of https://github.com/gosticks/openpose.git synced 2026-08-12 20:30:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

1.0.0rc3: Face+Debug+MPI render improved+minor bugs fixed (see release_notes)

Browse Source
This commit is contained in:
gineshidalgo99
2017-06-08 13:25:46 -04:00
parent cc4082b43a
commit 44398bd622
211 changed files with 5628 additions and 2304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/tutorial_wrapper/2_user_synchronous.cpp
+75 -61
View File
@@ -2,7 +2,7 @@
// Synchronous mode: ideal for performance. The user can add his own frames producer / post-processor / consumer to the OpenPose wrapper or use the default ones.
// This example shows the user how to use the OpenPose wrapper class:
// 1. Extract and render pose / heatmap / PAF of that image
// 1. Extract and render keypoint / heatmap / PAF of that image
// 2. Save the results on disc
// 3. Display the rendered pose
// Everything in a multi-thread scenario
@@ -20,11 +20,9 @@
#include <string> // std::string
#include <thread> // std::this_thread
#include <vector> // std::vector
// OpenCV dependencies
#include <opencv2/core/core.hpp> // cv::Mat & cv::Size
// Other 3rdpary depencencies
#include <gflags/gflags.h> // DEFINE_bool, DEFINE_int32, DEFINE_int64, DEFINE_uint64, DEFINE_double, DEFINE_string
#include <glog/logging.h> // google::InitGoogleLogging, CHECK, CHECK_EQ, LOG, VLOG, ...
#include <glog/logging.h> // google::InitGoogleLogging
// OpenPose dependencies
// Option a) Importing all modules
@@ -32,6 +30,7 @@
// Option b) Manually importing the desired modules. Recommended if you only intend to use a few modules.
// #include <openpose/core/headers.hpp>
// #include <openpose/experimental/headers.hpp>
// #include <openpose/face/headers.hpp>
// #include <openpose/filestream/headers.hpp>
// #include <openpose/gui/headers.hpp>
// #include <openpose/pose/headers.hpp>
@@ -40,56 +39,63 @@
// #include <openpose/utilities/headers.hpp>
// #include <openpose/wrapper/headers.hpp>
// Uncomment to avoid needing `op::` before each class and function of the OpenPose library. Analogously for OpenCV and the standard C++ library
// using namespace op;
// using namespace cv;
// using namespace std;
// Gflags in the command line terminal. Check all the options by adding the flag `--help`, e.g. `openpose.bin --help`.
// Note: This command will show you flags for several files. Check only the flags for the file you are checking. E.g. for `openpose.bin`, look for `Flags from examples/openpose/openpose.cpp:`.
// See all the available parameter options withe the `--help` flag. E.g. `./build/examples/openpose/openpose.bin --help`.
// Note: This command will show you flags for other unnecessary 3rdparty files. Check only the flags for the OpenPose
// executable. E.g. for `openpose.bin`, look for `Flags from examples/openpose/openpose.cpp:`.
// Debugging
DEFINE_int32(logging_level, 3, "The logging level. Integer in the range [0, 255]. 0 will output any log() message, while 255 will not output any."
" Current OpenPose library messages are in the range 0-4: 1 for low priority messages and 4 for important ones.");
// Producer
DEFINE_string(image_dir, "examples/media/", "Process a directory of images.");
// OpenPose
DEFINE_string(model_pose, "COCO", "Model to be used (e.g. COCO, MPI, MPI_4_layers).");
DEFINE_string(model_folder, "models/", "Folder where the pose models (COCO and MPI) are located.");
DEFINE_string(net_resolution, "656x368", "Multiples of 16.");
DEFINE_string(resolution, "1280x720", "The image resolution (display and output).");
DEFINE_int32(num_gpu, 1, "The number of GPU devices to use.");
DEFINE_string(resolution, "1280x720", "The image resolution (display and output). Use \"-1x-1\" to force the program to use the default images resolution.");
DEFINE_int32(num_gpu, -1, "The number of GPU devices to use. If negative, it will use all the available GPUs in your machine.");
DEFINE_int32(num_gpu_start, 0, "GPU device start number.");
DEFINE_int32(keypoint_scale, 0, "Scaling of the (x,y) coordinates of the final pose data array, i.e. the scale of the (x,y) coordinates that will be saved"
" with the `write_keypoint` & `write_keypoint_json` flags. Select `0` to scale it to the original source resolution, `1`to scale"
" it to the net output size (set with `net_resolution`), `2` to scale it to the final output size (set with `resolution`),"
" `3` to scale it in the range [0,1], and 4 for range [-1,1]. Non related with `num_scales` and `scale_gap`.");
// OpenPose Body Pose
DEFINE_string(model_pose, "COCO", "Model to be used (e.g. COCO, MPI, MPI_4_layers).");
DEFINE_string(net_resolution, "656x368", "Multiples of 16. If it is increased, the accuracy usually increases. If it is decreased, the speed increases.");
DEFINE_int32(num_scales, 1, "Number of scales to average.");
DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless num_scales>1. Initial scale is always 1. If you want to change the initial scale,"
" you actually want to multiply the `net_resolution` by your desired initial scale.");
DEFINE_int32(scale_mode, 0, "Scaling of the (x,y) coordinates of the final pose data array (op::Datum::pose), i.e. the scale of the (x,y) coordinates that"
" will be saved with the `write_pose` & `write_pose_json` flags. Select `0` to scale it to the original source resolution, `1`"
" to scale it to the net output size (set with `net_resolution`), `2` to scale it to the final output size (set with "
" `resolution`), `3` to scale it in the range [0,1], and 4 for range [-1,1]. Non related with `num_scales` and `scale_gap`.");
DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array (program speed will decrease). Not"
" required for our library, enable it only if you intend to process this information later. If more than one `add_heatmaps_X`"
" flag is enabled, it will place then in sequential memory order: body parts + bkg + PAFs. It will follow the order on"
" POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`.");
DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to background.");
DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs.");
DEFINE_int32(heatmaps_scale_mode, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [0,1], 1 for [-1,1]; and 2 for integer rounded [0,255].");
DEFINE_int32(heatmaps_scale, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [0,1], 1 for [-1,1]; and 2 for integer rounded [0,255].");
// OpenPose Face
DEFINE_bool(face, false, "Enables face keypoint detection. It will share some parameters from the body pose, e.g. `model_folder`.");
DEFINE_string(face_net_resolution, "368x368", "Multiples of 16. Analogous to `net_resolution` but applied to the face keypoint detector. 320x320 usually works fine"
" while giving a substantial speed up when multiple faces on the image.");
// OpenPose Hand
// OpenPose Rendering
DEFINE_bool(no_render_output, false, "If false, it will fill both `outputData` and `cvOutputData` with the original image + desired part to be shown."
" If true, it will leave them empty.");
DEFINE_int32(part_to_show, 0, "Part to show from the start.");
DEFINE_bool(disable_blending, false, "If false, it will blend the results with the original frame. If true, it will only display the results.");
DEFINE_bool(disable_blending, false, "If blending is enabled, it will merge the results with the original frame. If disabled, it will only display the results.");
// OpenPose Rendering Pose
DEFINE_bool(no_render_pose, false, "If false, it will fill both `outputData` and `cvOutputData` with the original image + desired part to be shown."
" If true, it will leave them empty.");
DEFINE_double(alpha_pose, 0.6, "Blending factor (range 0-1) for the body part rendering. 1 will show it completely, 0 will hide it.");
DEFINE_double(alpha_heatmap, 0.7, "Blending factor (range 0-1) between heatmap and original frame. 1 will only show the heatmap, 0 will only show the frame.");
// Consumer
// OpenPose Rendering Face
DEFINE_bool(no_render_face, false, "Analogous to `no_render_pose` but applied to the face keypoints and heat maps.");
DEFINE_double(alpha_face, 0.6, "Blending factor (range 0-1) for the body part rendering. 1 will show it completely, 0 will hide it.");
DEFINE_double(alpha_heatmap_face, 0.7, "Blending factor (range 0-1) between heatmap and original frame. 1 will only show the heatmap, 0 will only show the frame.");
// Result Saving
DEFINE_string(write_images, "", "Directory to write rendered frames in `write_images_format` image format.");
DEFINE_string(write_images_format, "png", "File extension and format for `write_images`, e.g. png, jpg or bmp. Check the OpenCV function cv::imwrite"
" for all compatible extensions.");
DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the final path does"
" not finish in `.avi`. It internally uses cv::VideoWriter.");
DEFINE_string(write_pose, "", "Directory to write the people pose data. Desired format on `write_pose_format`.");
DEFINE_string(write_pose_format, "yml", "File extension and format for `write_pose`: json, xml, yaml and yml. Json not available for OpenCV < 3.0,"
" use `write_pose_json` instead.");
DEFINE_string(write_pose_json, "", "Directory to write people pose data with *.json format, compatible with any OpenCV version.");
DEFINE_string(write_keypoint, "", "Directory to write the people body pose keypoint data. Desired format on `write_keypoint_format`.");
DEFINE_string(write_keypoint_format, "yml", "File extension and format for `write_keypoint`: json, xml, yaml and yml. Json not available for OpenCV < 3.0,"
" use `write_keypoint_json` instead.");
DEFINE_string(write_keypoint_json, "", "Directory to write people pose data with *.json format, compatible with any OpenCV version.");
DEFINE_string(write_coco_json, "", "Full file path to write people pose data with *.json COCO validation format.");
DEFINE_string(write_heatmaps, "", "Directory to write heatmaps with *.png format. At least 1 `add_heatmaps_X` flag must be enabled.");
DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`. Recommended `png` or any compressed and"
@@ -188,7 +194,7 @@ public:
{
// User's post-processing (after OpenPose processing & before OpenPose outputs) here
// datum.cvOutputData: rendered frame with pose or heatmaps
// datum.poseKeyPoints: Array<float> with the estimated pose
// datum.poseKeypoints: Array<float> with the estimated pose
try
{
if (datumsPtr != nullptr && !datumsPtr->empty())
@@ -216,7 +222,7 @@ public:
{
// User's displaying/saving/other processing here
// datum.cvOutputData: rendered frame with pose or heatmaps
// datum.poseKeyPoints: Array<float> with the estimated pose
// datum.poseKeypoints: Array<float> with the estimated pose
if (datumsPtr != nullptr && !datumsPtr->empty())
{
cv::imshow("User worker GUI", datumsPtr->at(0).cvOutputData);
@@ -248,18 +254,18 @@ op::PoseModel gflagToPoseModel(const std::string& poseModeString)
}
}
op::ScaleMode gflagToScaleMode(const int scaleMode)
op::ScaleMode gflagToScaleMode(const int keypointScale)
{
op::log("", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__);
if (scaleMode == 0)
if (keypointScale == 0)
return op::ScaleMode::InputResolution;
else if (scaleMode == 1)
else if (keypointScale == 1)
return op::ScaleMode::NetOutputResolution;
else if (scaleMode == 2)
else if (keypointScale == 2)
return op::ScaleMode::OutputResolution;
else if (scaleMode == 3)
else if (keypointScale == 3)
return op::ScaleMode::ZeroToOne;
else if (scaleMode == 4)
else if (keypointScale == 4)
return op::ScaleMode::PlusMinusOne;
else
{
@@ -283,27 +289,31 @@ std::vector<op::HeatMapType> gflagToHeatMaps(const bool heatmaps_add_parts, cons
}
// Google flags into program variables
std::tuple<cv::Size, cv::Size, op::PoseModel, op::ScaleMode, std::vector<op::HeatMapType>, op::ScaleMode> gflagsToOpParameters()
std::tuple<op::Point<int>, op::Point<int>, op::Point<int>, op::PoseModel, op::ScaleMode, std::vector<op::HeatMapType>, op::ScaleMode> gflagsToOpParameters()
{
op::log("", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__);
// outputSize
cv::Size outputSize;
auto nRead = sscanf(FLAGS_resolution.c_str(), "%dx%d", &outputSize.width, &outputSize.height);
op::Point<int> outputSize;
auto nRead = sscanf(FLAGS_resolution.c_str(), "%dx%d", &outputSize.x, &outputSize.y);
op::checkE(nRead, 2, "Error, resolution format (" + FLAGS_resolution + ") invalid, should be e.g., 960x540 ", __LINE__, __FUNCTION__, __FILE__);
// netInputSize
cv::Size netInputSize;
nRead = sscanf(FLAGS_net_resolution.c_str(), "%dx%d", &netInputSize.width, &netInputSize.height);
op::Point<int> netInputSize;
nRead = sscanf(FLAGS_net_resolution.c_str(), "%dx%d", &netInputSize.x, &netInputSize.y);
op::checkE(nRead, 2, "Error, net resolution format (" + FLAGS_net_resolution + ") invalid, should be e.g., 656x368 (multiples of 16)", __LINE__, __FUNCTION__, __FILE__);
// faceNetInputSize
op::Point<int> faceNetInputSize;
nRead = sscanf(FLAGS_face_net_resolution.c_str(), "%dx%d", &faceNetInputSize.x, &faceNetInputSize.y);
op::checkE(nRead, 2, "Error, face net resolution format (" + FLAGS_face_net_resolution + ") invalid, should be e.g., 368x368 (multiples of 16)", __LINE__, __FUNCTION__, __FILE__);
// poseModel
const auto poseModel = gflagToPoseModel(FLAGS_model_pose);
// scaleMode
const auto scaleMode = gflagToScaleMode(FLAGS_scale_mode);
// keypointScale
const auto keypointScale = gflagToScaleMode(FLAGS_keypoint_scale);
// heatmaps to add
const auto heatMapTypes = gflagToHeatMaps(FLAGS_heatmaps_add_parts, FLAGS_heatmaps_add_bkg, FLAGS_heatmaps_add_PAFs);
op::check(FLAGS_heatmaps_scale_mode >= 0 && FLAGS_heatmaps_scale_mode <= 2, "Non valid `heatmaps_scale_mode`.", __LINE__, __FUNCTION__, __FILE__);
const auto heatMapsScaleMode = (FLAGS_heatmaps_scale_mode == 0 ? op::ScaleMode::PlusMinusOne : (FLAGS_heatmaps_scale_mode == 1 ? op::ScaleMode::ZeroToOne : op::ScaleMode::UnsignedChar ));
op::check(FLAGS_heatmaps_scale >= 0 && FLAGS_heatmaps_scale <= 2, "Non valid `heatmaps_scale`.", __LINE__, __FUNCTION__, __FILE__);
const auto heatMapScale = (FLAGS_heatmaps_scale == 0 ? op::ScaleMode::PlusMinusOne : (FLAGS_heatmaps_scale == 1 ? op::ScaleMode::ZeroToOne : op::ScaleMode::UnsignedChar ));
// return
return std::make_tuple(outputSize, netInputSize, poseModel, scaleMode, heatMapTypes, heatMapsScaleMode);
return std::make_tuple(outputSize, netInputSize, faceNetInputSize, poseModel, keypointScale, heatMapTypes, heatMapScale);
}
int openPoseTutorialWrapper2()
@@ -317,13 +327,14 @@ int openPoseTutorialWrapper2()
const auto timerBegin = std::chrono::high_resolution_clock::now();
// Applying user defined configuration
cv::Size outputSize;
cv::Size netInputSize;
op::Point<int> outputSize;
op::Point<int> netInputSize;
op::Point<int> faceNetInputSize;
op::PoseModel poseModel;
op::ScaleMode scaleMode;
op::ScaleMode keypointScale;
std::vector<op::HeatMapType> heatMapTypes;
op::ScaleMode heatMapsScaleMode;
std::tie(outputSize, netInputSize, poseModel, scaleMode, heatMapTypes, heatMapsScaleMode) = gflagsToOpParameters();
op::ScaleMode heatMapScale;
std::tie(outputSize, netInputSize, faceNetInputSize, poseModel, keypointScale, heatMapTypes, heatMapScale) = gflagsToOpParameters();
op::log("", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__);
// Initializing the user custom classes
@@ -348,21 +359,24 @@ int openPoseTutorialWrapper2()
const bool displayGui = false;
const bool guiVerbose = false;
const bool fullScreen = false;
const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, scaleMode, FLAGS_num_gpu, FLAGS_num_gpu_start, FLAGS_num_scales, (float)FLAGS_scale_gap,
!FLAGS_no_render_output, poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap,
FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapsScaleMode};
const op::WrapperStructOutput wrapperStructOutput{displayGui, guiVerbose, fullScreen, FLAGS_write_pose, op::stringToDataFormat(FLAGS_write_pose_format),
FLAGS_write_pose_json, FLAGS_write_coco_json, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video,
FLAGS_write_heatmaps, FLAGS_write_heatmaps_format};
// Pose configuration (use WrapperStructPose{} for default and recommended configuration)
// Producer (use default to disable any input)
const op::WrapperStructPose wrapperStructPose{netInputSize, outputSize, keypointScale, FLAGS_num_gpu, FLAGS_num_gpu_start, FLAGS_num_scales, (float)FLAGS_scale_gap,
!FLAGS_no_render_pose, poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap,
FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale};
// (Experimental, function and class names might change in next versions) Face configuration (use op::WrapperStructFace{} to disable it)
const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, !FLAGS_no_render_face, (float)FLAGS_alpha_face, (float)FLAGS_alpha_heatmap_face};
// (Experimental, function and class names might change in next versions) Hand configuration (use op::WrapperStructFace{} to disable it)
const op::experimental::WrapperStructHand wrapperStructHand{false};
// Consumer (comment or use default argument to disable any output)
opWrapper.configure(wrapperStructPose, op::WrapperStructInput{}, wrapperStructOutput);
const op::WrapperStructOutput wrapperStructOutput{displayGui, guiVerbose, fullScreen, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format),
FLAGS_write_keypoint_json, FLAGS_write_coco_json, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video,
FLAGS_write_heatmaps, FLAGS_write_heatmaps_format};
// Configure wrapper
opWrapper.configure(wrapperStructPose, wrapperStructFace, wrapperStructHand, op::WrapperStructInput{}, wrapperStructOutput);
// Set to single-thread running (e.g. for debugging purposes)
// opWrapper.disableMultiThreading();
op::log("Starting thread(s)", op::Priority::Max);
// Two different ways of running the program on multithread enviroment
// Two different ways of running the program on multithread environment
// // Option a) Recommended - Also using the main thread (this thread) for processing (it saves 1 thread)
// // Start, run & stop threads
opWrapper.exec(); // It blocks this thread until all threads have finished