From b45dea108fddb648e5f2788a3287b4b559231731 Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Wed, 19 Dec 2018 13:28:40 -0500 Subject: [PATCH] Video can be saved from 3D results --- doc/demo_overview.md | 3 +- doc/quick_start.md | 4 +- doc/release_notes.md | 1 + examples/openpose/openpose.cpp | 4 +- .../1_custom_post_processing.cpp | 4 +- .../3_keypoints_from_image_configurable.cpp | 4 +- ...nchronous_loop_custom_input_and_output.cpp | 4 +- .../5_asynchronous_loop_custom_output.cpp | 4 +- .../6_synchronous_custom_postprocessing.cpp | 4 +- .../7_synchronous_custom_input.cpp | 4 +- .../8_synchronous_custom_output.cpp | 4 +- .../9_synchronous_custom_all.cpp | 4 +- include/openpose/core/datum.hpp | 5 ++ include/openpose/filestream/headers.hpp | 1 + include/openpose/filestream/wVideoSaver3D.hpp | 86 +++++++++++++++++++ include/openpose/flags.hpp | 4 +- include/openpose/gui/gui3D.hpp | 8 +- include/openpose/gui/wGui3D.hpp | 6 ++ include/openpose/wrapper/wrapperAuxiliary.hpp | 74 ++++++++++------ .../openpose/wrapper/wrapperStructOutput.hpp | 12 ++- src/openpose/filestream/defineTemplates.cpp | 1 + src/openpose/filestream/videoSaver.cpp | 3 + src/openpose/gui/gui3D.cpp | 38 +++++++- src/openpose/gui/guiAdam.cpp | 4 +- src/openpose/wrapper/wrapperStructOutput.cpp | 4 +- 25 files changed, 231 insertions(+), 59 deletions(-) create mode 100644 include/openpose/filestream/wVideoSaver3D.hpp diff --git a/doc/demo_overview.md b/doc/demo_overview.md index f6a1e7af..ad2449d5 100644 --- a/doc/demo_overview.md +++ b/doc/demo_overview.md @@ -233,6 +233,8 @@ Each flag is divided into flag name, default value, and description. - DEFINE_string(write_images_format, "png", "File extension and format for `write_images`, e.g., png, jpg or bmp. Check the OpenCV function cv::imwrite for all compatible extensions."); - DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the final path does not finish in `.avi`. It internally uses cv::VideoWriter. Flag `write_video_fps` controls FPS."); - DEFINE_double(write_video_fps, -1., "Frame rate for the recorded video. By default, it will try to get the input frames producer frame rate (e.g., input video or webcam frame rate). If the input frames producer does not have a set FPS (e.g., image_dir or webcam if OpenCV not compiled with its support), set this value accordingly (e.g., to the frame rate displayed by the OpenPose GUI)."); +- DEFINE_string(write_video_3d, "", "Analogous to `--write_video`, but applied to the 3D output."); +- DEFINE_string(write_video_adam, "", "Experimental, not available yet. Analogous to `--write_video`, but applied to Adam model."); - DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose keypoints (2-D and 3-D), as well as pose candidates (if `--part_candidates` enabled)."); - DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); - DEFINE_string(write_coco_foot_json, "", "Full file path to write people foot pose data with JSON COCO validation format."); @@ -243,7 +245,6 @@ Each flag is divided into flag name, default value, and description. - DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available for OpenCV < 3.0, use `write_json` instead."); 17. Result Saving - Extra Algorithms -- DEFINE_string(write_video_adam, "", "Experimental, not available yet. E.g., `~/Desktop/adamResult.avi`. Flag `write_video_fps` controls FPS."); - DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g., `~/Desktop/mocapResult.bvh`."); 18. UDP Communication diff --git a/doc/quick_start.md b/doc/quick_start.md index 3a28150f..9c346395 100644 --- a/doc/quick_start.md +++ b/doc/quick_start.md @@ -130,10 +130,10 @@ build\x64\Release\OpenPoseDemo.exe --flir_camera --3d --number_people_max 1 build\x64\Release\OpenPoseDemo.exe --flir_camera --3d --number_people_max 1 --face --hand ``` -2. Saving 3-D keypoints +2. Saving 3-D keypoints and video ``` # Ubuntu and Mac (same flags for Windows version) -./build/examples/openpose/openpose.bin --flir_camera --3d --number_people_max 1 --write_json output_folder_path/ +./build/examples/openpose/openpose.bin --flir_camera --3d --number_people_max 1 --write_json output_folder_path/ --write_video_3d output_folder_path/video_3d.avi ``` 3. Fast stereo camera image saving (without keypoint detection) for later post-processing diff --git a/doc/release_notes.md b/doc/release_notes.md index 6b3a5680..2e4f1397 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -295,6 +295,7 @@ OpenPose Library - Release Notes 25. Frame undistortion can be applied not only to FLIR cameras, but also to all other input sources (image, webcam, video, etc.). 26. Calibration improvements: 1. Improved chessboard orientation detection, more robust and less errors. + 27. Video with the 3D output can be saved with the new `--write_video_3d` flag. 2. Functions or parameters renamed: 1. By default, python example `tutorial_developer/python_2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368. 2. WrapperStructPose default parameters changed to match those of the OpenPose demo binary. diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index ba8aeebe..17ef3da5 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -101,8 +101,8 @@ int openPoseDemo() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapper.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_add_module/1_custom_post_processing.cpp b/examples/tutorial_add_module/1_custom_post_processing.cpp index de61ac67..65c2d2f1 100644 --- a/examples/tutorial_add_module/1_custom_post_processing.cpp +++ b/examples/tutorial_add_module/1_custom_post_processing.cpp @@ -114,8 +114,8 @@ int tutorialAddModule1() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp b/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp index 24f36a1e..c2fe8810 100644 --- a/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp +++ b/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp @@ -114,8 +114,8 @@ int tutorialApiCpp3() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp b/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp index e7692b92..e0102024 100644 --- a/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp +++ b/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp @@ -247,8 +247,8 @@ int tutorialApiCpp4() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp b/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp index f0629b71..104b6cb9 100644 --- a/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp +++ b/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp @@ -196,8 +196,8 @@ int tutorialApiCpp5() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp b/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp index 7f2a18b0..143cfe9d 100644 --- a/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp +++ b/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp @@ -164,8 +164,8 @@ int tutorialApiCpp6() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp b/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp index 024e4bd6..8e61c20e 100644 --- a/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp +++ b/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp @@ -193,8 +193,8 @@ int tutorialApiCpp7() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp b/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp index 68fca483..2f2ac05a 100644 --- a/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp +++ b/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp @@ -211,8 +211,8 @@ int tutorialApiCpp8() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp b/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp index 6a3ebcbb..291ca08b 100644 --- a/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp +++ b/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp @@ -304,8 +304,8 @@ int tutorialApiCpp9() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, - FLAGS_udp_port}; + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/include/openpose/core/datum.hpp b/include/openpose/core/datum.hpp index 7d7e759a..472f6ab2 100644 --- a/include/openpose/core/datum.hpp +++ b/include/openpose/core/datum.hpp @@ -70,6 +70,11 @@ namespace op */ cv::Mat cvOutputData; + /** + * Rendered 3D image in cv::Mat uchar format. + */ + cv::Mat cvOutputData3D; + // ------------------------------ Resulting Array data parameters ------------------------------ // /** * Body pose (x,y,score) locations for each person in the image. diff --git a/include/openpose/filestream/headers.hpp b/include/openpose/filestream/headers.hpp index 9de83820..68db0bc5 100644 --- a/include/openpose/filestream/headers.hpp +++ b/include/openpose/filestream/headers.hpp @@ -24,5 +24,6 @@ #include #include #include +#include #endif // OPENPOSE_FILESTREAM_HEADERS_HPP diff --git a/include/openpose/filestream/wVideoSaver3D.hpp b/include/openpose/filestream/wVideoSaver3D.hpp new file mode 100644 index 00000000..c1109a2c --- /dev/null +++ b/include/openpose/filestream/wVideoSaver3D.hpp @@ -0,0 +1,86 @@ +#ifndef OPENPOSE_FILESTREAM_W_VIDEO_SAVER_3D_HPP +#define OPENPOSE_FILESTREAM_W_VIDEO_SAVER_3D_HPP + +#include +#include +#include + +namespace op +{ + template + class WVideoSaver3D : public WorkerConsumer + { + public: + explicit WVideoSaver3D(const std::shared_ptr& videoSaver); + + virtual ~WVideoSaver3D(); + + void initializationOnThread(); + + void workConsumer(const TDatums& tDatums); + + private: + std::shared_ptr spVideoSaver; + + DELETE_COPY(WVideoSaver3D); + }; +} + + + + + +// Implementation +#include +namespace op +{ + template + WVideoSaver3D::WVideoSaver3D(const std::shared_ptr& videoSaver) : + spVideoSaver{videoSaver} + { + } + + template + WVideoSaver3D::~WVideoSaver3D() + { + } + + template + void WVideoSaver3D::initializationOnThread() + { + } + + template + void WVideoSaver3D::workConsumer(const TDatums& tDatums) + { + try + { + if (checkNoNullNorEmpty(tDatums)) + { + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + // T* to T + auto& tDatumsNoPtr = *tDatums; + // Record video(s) + if (!tDatumsNoPtr.empty()) + spVideoSaver->write(tDatumsNoPtr[0].cvOutputData3D); + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + } + catch (const std::exception& e) + { + this->stop(); + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + COMPILE_TEMPLATE_DATUM(WVideoSaver3D); +} + +#endif // OPENPOSE_FILESTREAM_W_VIDEO_SAVER_3D_HPP diff --git a/include/openpose/flags.hpp b/include/openpose/flags.hpp index 7d722649..311d30e9 100644 --- a/include/openpose/flags.hpp +++ b/include/openpose/flags.hpp @@ -219,6 +219,8 @@ DEFINE_double(write_video_fps, -1., "Frame rate for the reco " frame rate (e.g., input video or webcam frame rate). If the input frames producer does not" " have a set FPS (e.g., image_dir or webcam if OpenCV not compiled with its support), set" " this value accordingly (e.g., to the frame rate displayed by the OpenPose GUI)."); +DEFINE_string(write_video_3d, "", "Analogous to `--write_video`, but applied to the 3D output."); +DEFINE_string(write_video_adam, "", "Experimental, not available yet. Analogous to `--write_video`, but applied to Adam model."); DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose" " keypoints (2-D and 3-D), as well as pose candidates (if `--part_candidates` enabled)."); DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); @@ -235,8 +237,6 @@ DEFINE_string(write_keypoint, "", "(Deprecated, use `write DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml," " yaml & yml. Json not available for OpenCV < 3.0, use `write_json` instead."); // Result Saving - Extra Algorithms -DEFINE_string(write_video_adam, "", "Experimental, not available yet. E.g., `~/Desktop/adamResult.avi`. Flag `write_video_fps`" - " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g., `~/Desktop/mocapResult.bvh`."); // UDP Communication DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); diff --git a/include/openpose/gui/gui3D.hpp b/include/openpose/gui/gui3D.hpp index d0a49b36..ed298d32 100644 --- a/include/openpose/gui/gui3D.hpp +++ b/include/openpose/gui/gui3D.hpp @@ -20,7 +20,8 @@ namespace op const std::vector>& handExtractorNets = {}, const std::vector>& renderers = {}, const PoseModel poseModel = PoseModel::BODY_25, - const DisplayMode displayMode = DisplayMode::DisplayAll); + const DisplayMode displayMode = DisplayMode::DisplayAll, + const bool copyGlToCvMat = false); virtual ~Gui3D(); @@ -30,6 +31,11 @@ namespace op const Array& leftHandKeypoints3D, const Array& rightHandKeypoints3D); virtual void update(); + + virtual cv::Mat readCvMat(); + + private: + const bool mCopyGlToCvMat; }; } diff --git a/include/openpose/gui/wGui3D.hpp b/include/openpose/gui/wGui3D.hpp index 74d56fd6..a1b9d87d 100644 --- a/include/openpose/gui/wGui3D.hpp +++ b/include/openpose/gui/wGui3D.hpp @@ -86,6 +86,12 @@ namespace op } // Refresh/update GUI spGui3D->update(); + // Read OpenCV mat equivalent + if (!tDatums->empty()) + { + auto& tDatum = (*tDatums)[0]; + tDatum.cvOutputData3D = spGui3D->readCvMat(); + } // Profiling speed if (!tDatums->empty()) { diff --git a/include/openpose/wrapper/wrapperAuxiliary.hpp b/include/openpose/wrapper/wrapperAuxiliary.hpp index 6def0751..09c924f1 100644 --- a/include/openpose/wrapper/wrapperAuxiliary.hpp +++ b/include/openpose/wrapper/wrapperAuxiliary.hpp @@ -645,8 +645,9 @@ namespace op wrapperStructOutput.writeImagesFormat); outputWs.emplace_back(std::make_shared>(imageSaver)); } - // Write frames as *.avi video on hard disk - if (!wrapperStructOutput.writeVideo.empty() || !wrapperStructOutput.writeBvh.empty()) + auto originalVideoFps = 0.; + if (!wrapperStructOutput.writeVideo.empty() || !wrapperStructOutput.writeVideo3D.empty() + || !wrapperStructOutput.writeBvh.empty()) { if (wrapperStructOutput.writeVideoFps <= 0 && (!oPProducer || producerSharedPtr->get(CV_CAP_PROP_FPS) <= 0)) @@ -655,35 +656,36 @@ namespace op " will have to know or guess the frame rate; if it is a webcam, you should use the OpenPose" " displayed FPS as desired value. If you do not care, simply add `--write_video_fps 30`.", __LINE__, __FUNCTION__, __FILE__); - const auto originalVideoFps = ( + originalVideoFps = ( wrapperStructOutput.writeVideoFps > 0 ? wrapperStructOutput.writeVideoFps : producerSharedPtr->get(CV_CAP_PROP_FPS)); - if (!wrapperStructOutput.writeVideo.empty()) - { - if (!oPProducer) - error("Video file can only be recorded inside `wrapper/wrapper.hpp` if the producer" - " is one of the default ones (e.g., video, webcam, ...).", - __LINE__, __FUNCTION__, __FILE__); - const auto videoSaver = std::make_shared( - wrapperStructOutput.writeVideo, CV_FOURCC('M','J','P','G'), originalVideoFps); - outputWs.emplace_back(std::make_shared>(videoSaver)); - } - // Write joint angles as *.bvh file on hard disk -#ifdef USE_3D_ADAM_MODEL - if (!wrapperStructOutput.writeBvh.empty()) - { - const auto bvhSaver = std::make_shared( - wrapperStructOutput.writeBvh, JointAngleEstimation::getTotalModel(), originalVideoFps - ); - outputWs.emplace_back(std::make_shared>(bvhSaver)); - } -#endif } + // Write frames as *.avi video on hard disk + if (!wrapperStructOutput.writeVideo.empty()) + { + if (!oPProducer) + error("Video file can only be recorded inside `wrapper/wrapper.hpp` if the producer" + " is one of the default ones (e.g., video, webcam, ...).", + __LINE__, __FUNCTION__, __FILE__); + const auto videoSaver = std::make_shared( + wrapperStructOutput.writeVideo, CV_FOURCC('M','J','P','G'), originalVideoFps); + outputWs.emplace_back(std::make_shared>(videoSaver)); + } + // Write joint angles as *.bvh file on hard disk +#ifdef USE_3D_ADAM_MODEL + if (!wrapperStructOutput.writeBvh.empty()) + { + const auto bvhSaver = std::make_shared( + wrapperStructOutput.writeBvh, JointAngleEstimation::getTotalModel(), originalVideoFps + ); + outputWs.emplace_back(std::make_shared>(bvhSaver)); + } +#endif // Write heat maps as desired image format on hard disk if (!writeHeatMapsCleaned.empty()) { - const auto heatMapSaver = std::make_shared(writeHeatMapsCleaned, - wrapperStructOutput.writeHeatMapsFormat); + const auto heatMapSaver = std::make_shared( + writeHeatMapsCleaned, wrapperStructOutput.writeHeatMapsFormat); outputWs.emplace_back(std::make_shared>(heatMapSaver)); } // Add frame information for GUI @@ -699,6 +701,7 @@ namespace op } // Minimal graphical user interface (GUI) TWorker guiW; + TWorker videoSaver3DW; if (guiEnabled) { // PoseRenderers to Renderers @@ -726,6 +729,10 @@ namespace op ); // WGui guiW = {std::make_shared>(gui)}; + // Write 3D frames as *.avi video on hard disk + if (!wrapperStructOutput.writeVideo3D.empty()) + error("3D video can only be recorded if 3D render is enabled.", + __LINE__, __FUNCTION__, __FILE__); #endif } // 3-D (+2-D) display @@ -736,10 +743,18 @@ namespace op const auto gui = std::make_shared( finalOutputSizeGui, wrapperStructGui.fullScreen, threadManager.getIsRunningSharedPtr(), spVideoSeek, poseExtractorNets, faceExtractorNets, handExtractorNets, renderers, - wrapperStructPose.poseModel, wrapperStructGui.displayMode + wrapperStructPose.poseModel, wrapperStructGui.displayMode, + !wrapperStructOutput.writeVideo3D.empty() ); // WGui guiW = {std::make_shared>(gui)}; + // Write 3D frames as *.avi video on hard disk + if (!wrapperStructOutput.writeVideo3D.empty()) + { + const auto videoSaver = std::make_shared( + wrapperStructOutput.writeVideo3D, CV_FOURCC('M','J','P','G'), originalVideoFps); + videoSaver3DW = std::make_shared>(videoSaver); + } } // 2-D display else if (wrapperStructGui.displayMode == DisplayMode::Display2D) @@ -751,6 +766,10 @@ namespace op ); // WGui guiW = {std::make_shared>(gui)}; + // Write 3D frames as *.avi video on hard disk + if (!wrapperStructOutput.writeVideo3D.empty()) + error("3D video can only be recorded if 3D render is enabled.", + __LINE__, __FUNCTION__, __FILE__); } else error("Unknown DisplayMode.", __LINE__, __FUNCTION__, __FILE__); @@ -989,6 +1008,9 @@ namespace op // Thread Y+1, queues Q+1 -> Q+2 log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); threadManager.add(threadId, guiW, queueIn++, queueOut++); + // Saving 3D output + if (videoSaver3DW != nullptr) + threadManager.add(threadId, videoSaver3DW, queueIn++, queueOut++); threadIdPP(threadId, multiThreadEnabled); } log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); diff --git a/include/openpose/wrapper/wrapperStructOutput.hpp b/include/openpose/wrapper/wrapperStructOutput.hpp index 30d5762c..1f7f2898 100644 --- a/include/openpose/wrapper/wrapperStructOutput.hpp +++ b/include/openpose/wrapper/wrapperStructOutput.hpp @@ -104,6 +104,13 @@ namespace op */ double writeVideoFps; + /** + * Rendered 3D images saving video path. + * Please, use *.avi format. + * If it is empty (default), it is disabled. + */ + std::string writeVideo3D; + /** * Rendered Adam images saving video path. * Please, use *.avi format. @@ -140,8 +147,9 @@ namespace op const int writeCocoJsonVariant = 1, const std::string& writeImages = "", const std::string& writeImagesFormat = "", const std::string& writeVideo = "", const double writeVideoFps = -1., const std::string& writeHeatMaps = "", - const std::string& writeHeatMapsFormat = "", const std::string& writeVideoAdam = "", - const std::string& writeBvh = "", const std::string& udpHost = "", const std::string& udpPort = ""); + const std::string& writeHeatMapsFormat = "", const std::string& writeVideo3D = "", + const std::string& writeVideoAdam = "", const std::string& writeBvh = "", + const std::string& udpHost = "", const std::string& udpPort = ""); }; } diff --git a/src/openpose/filestream/defineTemplates.cpp b/src/openpose/filestream/defineTemplates.cpp index 8effda41..c67366ed 100644 --- a/src/openpose/filestream/defineTemplates.cpp +++ b/src/openpose/filestream/defineTemplates.cpp @@ -14,4 +14,5 @@ namespace op DEFINE_TEMPLATE_DATUM(WPoseSaver); DEFINE_TEMPLATE_DATUM(WUdpSender); DEFINE_TEMPLATE_DATUM(WVideoSaver); + DEFINE_TEMPLATE_DATUM(WVideoSaver3D); } diff --git a/src/openpose/filestream/videoSaver.cpp b/src/openpose/filestream/videoSaver.cpp index 02e58aee..d84e5f7b 100644 --- a/src/openpose/filestream/videoSaver.cpp +++ b/src/openpose/filestream/videoSaver.cpp @@ -85,6 +85,9 @@ namespace op // Sanity check if (cvMats.empty()) error("The image(s) to be saved cannot be empty.", __LINE__, __FUNCTION__, __FILE__); + for (const auto& cvMat : cvMats) + if (cvMat.empty()) + error("The image(s) to be saved cannot be empty.", __LINE__, __FUNCTION__, __FILE__); // Open video (1st frame) // Done here and not in the constructor to handle cases where the resolution is not known (e.g., // reading images or multiple cameras) diff --git a/src/openpose/gui/gui3D.cpp b/src/openpose/gui/gui3D.cpp index 493363ce..b725cdae 100644 --- a/src/openpose/gui/gui3D.cpp +++ b/src/openpose/gui/gui3D.cpp @@ -17,6 +17,8 @@ namespace op { #ifdef USE_3D_RENDERER const bool LOG_VERBOSE_3D_RENDERER = false; + const auto WINDOW_WIDTH = 1280; + const auto WINDOW_HEIGHT = 720; std::atomic sConstructorSet{false}; struct Keypoints3D @@ -384,7 +386,7 @@ namespace op int my_argc = 0; glutInit(&my_argc, my_argv); // Setup the size, position, and display mode for new windows - glutInitWindowSize(1280, 720); + glutInitWindowSize(WINDOW_WIDTH, WINDOW_HEIGHT); glutInitWindowPosition(200, 0); // glutSetOption(GLUT_MULTISAMPLE,8); // Ideally adding also GLUT_BORDERLESS | GLUT_CAPTIONLESS should fix the problem of disabling the `x` @@ -421,9 +423,10 @@ namespace op const std::vector>& faceExtractorNets, const std::vector>& handExtractorNets, const std::vector>& renderers, const PoseModel poseModel, - const DisplayMode displayMode) : + const DisplayMode displayMode, const bool copyGlToCvMat) : Gui{outputSize, fullScreen, isRunningSharedPtr, videoSeekSharedPtr, poseExtractorNets, faceExtractorNets, - handExtractorNets, renderers, displayMode} + handExtractorNets, renderers, displayMode}, + mCopyGlToCvMat{copyGlToCvMat} { try { @@ -522,7 +525,7 @@ namespace op void Gui3D::update() { try - { + { // 2-D rendering // Display all 2-D views if (mDisplayMode == DisplayMode::DisplayAll || mDisplayMode == DisplayMode::Display2D) @@ -552,4 +555,31 @@ namespace op error(e.what(), __LINE__, __FUNCTION__, __FILE__); } } + + cv::Mat Gui3D::readCvMat() + { + try + { + // 3-D rendering + #ifdef USE_3D_RENDERER + cv::Mat image; + if (mDisplayMode == DisplayMode::DisplayAll || mDisplayMode == DisplayMode::Display3D) + { + // Save/display 3D display in OpenCV window + if (mCopyGlToCvMat) + { + image = cv::Mat(WINDOW_HEIGHT, WINDOW_WIDTH, CV_8UC3); + glReadPixels(0, 0, WINDOW_WIDTH, WINDOW_HEIGHT, GL_BGR, GL_UNSIGNED_BYTE, image.data); + cv::flip(image, image, 0); + } + } + return image; + #endif + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return cv::Mat(); + } + } } diff --git a/src/openpose/gui/guiAdam.cpp b/src/openpose/gui/guiAdam.cpp index 9a872302..a1f23158 100644 --- a/src/openpose/gui/guiAdam.cpp +++ b/src/openpose/gui/guiAdam.cpp @@ -234,7 +234,7 @@ namespace op spImpl->spRender->RenderAndRead(); // read the image into read_buffer // spImpl->spRender->Display(); - // Save/display Adam display in opencv window + // Save/display Adam display in OpenCV window if (!spImpl->mWriteAdamRenderAsVideo.empty()) { // img is y-flipped, and in RGB order @@ -247,7 +247,7 @@ namespace op const auto originalVideoFps = 30; spImpl->spVideoSaver = std::make_shared( spImpl->mWriteAdamRenderAsVideo, CV_FOURCC('M','J','P','G'), - originalVideoFps, Point{img.cols, img.rows} + originalVideoFps ); } spImpl->spVideoSaver->write(img); diff --git a/src/openpose/wrapper/wrapperStructOutput.cpp b/src/openpose/wrapper/wrapperStructOutput.cpp index 504682c5..7ab0150c 100644 --- a/src/openpose/wrapper/wrapperStructOutput.cpp +++ b/src/openpose/wrapper/wrapperStructOutput.cpp @@ -7,7 +7,8 @@ namespace op const std::string& writeJson_, const std::string& writeCocoJson_, const std::string& writeCocoFootJson_, const int writeCocoJsonVariant_, const std::string& writeImages_, const std::string& writeImagesFormat_, const std::string& writeVideo_, const double writeVideoFps_, const std::string& writeHeatMaps_, - const std::string& writeHeatMapsFormat_, const std::string& writeVideoAdam_, const std::string& writeBvh_, + const std::string& writeHeatMapsFormat_, const std::string& writeVideo3D_, + const std::string& writeVideoAdam_, const std::string& writeBvh_, const std::string& udpHost_, const std::string& udpPort_) : verbose{verbose_}, writeKeypoint{writeKeypoint_}, @@ -22,6 +23,7 @@ namespace op writeHeatMaps{writeHeatMaps_}, writeHeatMapsFormat{writeHeatMapsFormat_}, writeVideoFps{writeVideoFps_}, + writeVideo3D{writeVideo3D_}, writeVideoAdam{writeVideoAdam_}, writeBvh{writeBvh_}, udpHost{udpHost_},