From f0664bcd0f4ae86fb964a670ce1ca282b69cba20 Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Thu, 17 Jan 2019 18:26:32 -0500 Subject: [PATCH] Added mp4 saving option --- .travis.yml | 62 +++++--- CMakeLists.txt | 6 +- README.md | 11 +- doc/contributors.md | 2 +- doc/demo_overview.md | 5 +- doc/release_notes.md | 19 ++- examples/openpose/openpose.cpp | 4 +- .../1_custom_post_processing.cpp | 4 +- .../3_keypoints_from_image_configurable.cpp | 4 +- ...nchronous_loop_custom_input_and_output.cpp | 4 +- .../5_asynchronous_loop_custom_output.cpp | 4 +- .../6_synchronous_custom_postprocessing.cpp | 4 +- .../7_synchronous_custom_input.cpp | 4 +- .../8_synchronous_custom_output.cpp | 4 +- .../9_synchronous_custom_all.cpp | 4 +- include/openpose/filestream/imageSaver.hpp | 2 + include/openpose/filestream/videoSaver.hpp | 17 +- include/openpose/flags.hpp | 10 +- include/openpose/utilities/fileSystem.hpp | 1 + include/openpose/wrapper/wrapperAuxiliary.hpp | 11 +- .../openpose/wrapper/wrapperStructOutput.hpp | 32 ++-- python/openpose/openpose_python.cpp | 14 +- scripts/travis/configure_cmake.sh | 5 + scripts/travis/defaults.sh | 4 +- src/openpose/filestream/imageSaver.cpp | 12 ++ src/openpose/filestream/videoSaver.cpp | 149 ++++++++++++++++-- src/openpose/gui/guiAdam.cpp | 2 +- src/openpose/net/bodyPartConnectorBase.cpp | 22 +-- src/openpose/wrapper/wrapperStructOutput.cpp | 11 +- 29 files changed, 311 insertions(+), 122 deletions(-) diff --git a/.travis.yml b/.travis.yml index d34836e8..d3e278c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,31 +22,61 @@ matrix: # WITH_MKL: false include: # Ubuntu 16.04 - # Ubuntu 16.04 - Default - CMake - CUDA + # Ubuntu 16.04 - Python - CMake - CUDA - os: linux dist: xenial - env: NAME="U16-default-cmake-cuda8" + env: NAME="U16-python-cmake-cuda8" WITH_PYTHON=true sudo: required # Generate and deploy documentation after_success: - cd $TRAVIS_BUILD_DIR - chmod +x scripts/generate_gh_pages.sh - ./scripts/generate_gh_pages.sh - # Ubuntu 16.04 - Python - CMake - CUDA + # Ubuntu 16.04 - Python - CMake - CPU - os: linux dist: xenial - env: NAME="U16-python-cmake-cuda8" WITH_PYTHON=true + env: NAME="U16-python-cmake-cpu" WITH_PYTHON=true WITH_CUDA=false + sudo: required + # Ubuntu 16.04 - Default - CMake - CUDA + - os: linux + dist: xenial + env: NAME="U16-default-cmake-cuda8" sudo: required # Ubuntu 16.04 - Default - CMake - CPU - os: linux dist: xenial env: NAME="U16-default-cmake-cpu" WITH_CUDA=false sudo: required - # Ubuntu 16.04 - Python - CMake - CPU + # Ubuntu 16.04 - Python - CMake - CPU - Debug - os: linux dist: xenial - env: NAME="U16-python-cmake-cpu" WITH_PYTHON=true WITH_CUDA=false + env: NAME="U16-python-cmake-cpu-debug" WITH_PYTHON=true WITH_CUDA=false WITH_DEBUG=true sudo: required + + # Mac OSX + # Mac OSX - Python - CMake - CPU + - os: osx + osx_image: xcode9.4 # xcode10.1 does not work with Python # Versions: https://docs.travis-ci.com/user/languages/objective-c#supported-xcode-versions + env: NAME="OSX-python-cmake-cpu" WITH_CUDA=false WITH_PYTHON=true + sudo: required + # Mac OSX - Default - CMake - CPU + - os: osx + osx_image: xcode10.1 # Versions: https://docs.travis-ci.com/user/languages/objective-c#supported-xcode-versions + env: NAME="OSX-default-cmake-cpu" WITH_CUDA=false + sudo: required + # Mac OSX - Python - CMake - CPU - Debug + - os: osx + osx_image: xcode9.4 # xcode10.1 does not work with Python # Versions: https://docs.travis-ci.com/user/languages/objective-c#supported-xcode-versions + env: NAME="OSX-python-cmake-cpu" WITH_CUDA=false WITH_PYTHON=true WITH_DEBUG=true + sudo: required + + # # TO-DO: To be implemented + # # Windows + # # Windows - Default - CMake - CUDA + # - os: windows + # env: NAME="W10-default-cmake-cuda8" + + # Ubuntu (others) # Ubuntu 16.04 - Default - Make - CUDA - os: linux dist: xenial @@ -74,8 +104,6 @@ matrix: # dist: xenial # env: NAME="U16-default-cmake-cuda8-nocudnn" WITH_CUDNN=false # sudo: required - - # Ubuntu 14.04 # Ubuntu 14.04 - Default - CMake - CPU - os: linux dist: trusty @@ -93,24 +121,6 @@ matrix: # env: NAME="U14-default-cmake-cuda8" # sudo: required - # # TO-DO: To be implemented - # # Windows - # # Windows - Default - CMake - CUDA - # - os: windows - # env: NAME="W10-default-cmake-cuda8" - - # Mac OSX - # Mac OSX - Default - CMake - CPU - - os: osx - osx_image: xcode10.1 # Versions: https://docs.travis-ci.com/user/languages/objective-c#supported-xcode-versions - env: NAME="OSX-default-cmake-cpu" WITH_CUDA=false - sudo: required - - - os: osx - osx_image: xcode9.4 # xcode10.1 does not work with Python # Versions: https://docs.travis-ci.com/user/languages/objective-c#supported-xcode-versions - env: NAME="OSX-python-cmake-cpu" WITH_CUDA=false WITH_PYTHON=true - sudo: required - # Install apt dependencies addons: apt: diff --git a/CMakeLists.txt b/CMakeLists.txt index 126b98b6..60520d16 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -648,7 +648,8 @@ if (UNIX OR APPLE) # Build Process set(CAFFE_CPU_ONLY OFF) if (${GPU_MODE} MATCHES "CUDA") - execute_process(COMMAND git checkout master WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/caffe) + # execute_process(COMMAND git checkout master WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/caffe) + execute_process(COMMAND git checkout b5ede48 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/caffe) elseif (${GPU_MODE} MATCHES "CPU_ONLY") if (USE_MKL) #execute_process(COMMAND git checkout intel WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/caffe) @@ -683,7 +684,8 @@ if (UNIX OR APPLE) "${CMAKE_BINARY_DIR}/caffe/lib/libmkldnn.so" ) else (USE_MKL) - execute_process(COMMAND git checkout master WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/caffe) + # execute_process(COMMAND git checkout master WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/caffe) + execute_process(COMMAND git checkout b5ede48 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/caffe) endif (USE_MKL) set(CAFFE_CPU_ONLY ON) set(USE_CUDNN OFF) diff --git a/README.md b/README.md index ebf61461..8532eacd 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,14 @@ ----------------- -| **`Linux`** | -|-------------| -|[![Build Status](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose.svg?branch=master)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose)| +| | Python (CUDA GPU) | Python (CPU)| CUDA GPU | CPU | Debug mode | +| :---: | :---: | :---: | :---: |:---: | :---: | +| **Linux** | [![Status](https://travis-matrix-badges.herokuapp.com/repos/CMU-Perceptual-Computing-Lab/openpose/branches/master/1)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose) | [![Status](https://travis-matrix-badges.herokuapp.com/repos/CMU-Perceptual-Computing-Lab/openpose/branches/master/2)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose) | [![Status](https://travis-matrix-badges.herokuapp.com/repos/CMU-Perceptual-Computing-Lab/openpose/branches/master/3)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose) | [![Status](https://travis-matrix-badges.herokuapp.com/repos/CMU-Perceptual-Computing-Lab/openpose/branches/master/4)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose) | [![Status](https://travis-matrix-badges.herokuapp.com/repos/CMU-Perceptual-Computing-Lab/openpose/branches/master/5)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose) | +| **MacOS** | | [![Status](https://travis-matrix-badges.herokuapp.com/repos/CMU-Perceptual-Computing-Lab/openpose/branches/master/6)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose) | | [![Status](https://travis-matrix-badges.herokuapp.com/repos/CMU-Perceptual-Computing-Lab/openpose/branches/master/7)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose) | [![Status](https://travis-matrix-badges.herokuapp.com/repos/CMU-Perceptual-Computing-Lab/openpose/branches/master/8)](https://travis-ci.org/CMU-Perceptual-Computing-Lab/openpose) | + + [OpenPose](https://github.com/CMU-Perceptual-Computing-Lab/openpose) represents the **first real-time multi-person system to jointly detect human body, hand, facial, and foot keypoints (in total 135 keypoints) on single images**. diff --git a/doc/contributors.md b/doc/contributors.md index b8d7fd65..afc852d7 100644 --- a/doc/contributors.md +++ b/doc/contributors.md @@ -12,7 +12,7 @@ OpenPose is authored by [Gines Hidalgo](https://www.gineshidalgo.com/), [Zhe Cao We would also like to thank the following people who have highly contributed to OpenPose: 1. [Yaadhav Raaj](https://www.linkedin.com/in/yaadhavraaj): OpenPose maintainer, CPU version, OpenCL version, Mac version, Python API, and person tracker. -2. [Bikramjot Hanzra](https://www.linkedin.com/in/bikz05): Former OpenPose maintainer, CMake (Ubuntu and Windows) version, and initial Travis Build version. +2. [Bikramjot Hanzra](https://www.linkedin.com/in/bikz05): Former OpenPose maintainer, CMake (Ubuntu and Windows) version, and initial Travis Build version for Ubuntu. 3. [Donglai Xiang](https://xiangdonglai.github.io): Camera calibration toolbox improvement, including the implementation of its bundle adjustment algorithm. 4. [Luis Fernando Fraga](https://github.com/fragalfernando): Implementation of Lukas-Kanade algorith and person ID extractor. 5. [Helen Medina](https://github.com/helen-medina): Initial Windows version. diff --git a/doc/demo_overview.md b/doc/demo_overview.md index 4b5f819f..f27ce606 100644 --- a/doc/demo_overview.md +++ b/doc/demo_overview.md @@ -127,7 +127,7 @@ We enumerate some of the most important flags, check the `Flags Detailed Descrip -## Flags Description +## Flag Description Each flag is divided into flag name, default value, and description. 1. Debugging/Other @@ -233,8 +233,9 @@ Each flag is divided into flag name, default value, and description. 16. Result Saving - DEFINE_string(write_images, "", "Directory to write rendered frames in `write_images_format` image format."); - DEFINE_string(write_images_format, "png", "File extension and format for `write_images`, e.g., png, jpg or bmp. Check the OpenCV function cv::imwrite for all compatible extensions."); -- DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the final path does not finish in `.avi`. It internally uses cv::VideoWriter. Flag `write_video_fps` controls FPS."); +- DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the final path does not finish in `.avi`. It internally uses cv::VideoWriter. Flag `write_video_fps` controls FPS. Alternatively, the video extension can be `.mp4`, resulting in a file with a much smaller size and allowing `--write_video_with_audio`. However, that would require: 1) Ubuntu or Mac system, 2) FFmpeg library installed (`sudo apt-get install ffmpeg`), 3) the creation temporarily of a folder with the same file path than the final video (without the extension) to storage the intermediate frames that will later be used to generate the final MP4 video."); - DEFINE_double(write_video_fps, -1., "Frame rate for the recorded video. By default, it will try to get the input frames producer frame rate (e.g., input video or webcam frame rate). If the input frames producer does not have a set FPS (e.g., image_dir or webcam if OpenCV not compiled with its support), set this value accordingly (e.g., to the frame rate displayed by the OpenPose GUI)."); +- DEFINE_bool(write_video_with_audio, false, "If the input is video and the output is so too, it will save the video with audio. It requires the output video file path finishing in `.mp4` format (see `write_video` for details)."); - DEFINE_string(write_video_3d, "", "Analogous to `--write_video`, but applied to the 3D output."); - DEFINE_string(write_video_adam, "", "Experimental, not available yet. Analogous to `--write_video`, but applied to Adam model."); - DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose keypoints (2-D and 3-D), as well as pose candidates (if `--part_candidates` enabled)."); diff --git a/doc/release_notes.md b/doc/release_notes.md index 57989bdd..57554e16 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -290,7 +290,11 @@ OpenPose Library - Release Notes 20. Added flag `--verbose` to plot the progress. 21. Added find_package(Protobuf) to allow specific versions of Protobuf. 22. Examples do not end in core dumped if an OpenPose exception occurred during initialization, but it is rather closed returning -1. However, it will still results in core dumped if the exception occurs during multi-threading execution. - 23. Video (`--write_video`) can be generated from images (`--image_dir`), as long as they maintain the same resolution. + 23. Video saving improvements: + 1. Video (`--write_video`) can be generated from images (`--image_dir`), as long as they maintain the same resolution. + 2. Video with the 3D output can be saved with the new `--write_video_3d` flag. + 3. Added the capability of saving videos in MP4 format (by using the ffmpeg library). + 4. Added the flag `write_video_with_audio` to enable saving these output MP4 videos with audio. 24. Added `--fps_max` flag to limit the maximum processing frame rate of OpenPose (useful to display results at a maximum desired speed). 25. Frame undistortion can be applied not only to FLIR cameras, but also to all other input sources (image, webcam, video, etc.). 26. Calibration improvements: @@ -298,13 +302,12 @@ OpenPose Library - Release Notes 2. Triangulation functions (triangulate and triangulateWithOptimization) public, so calibration can use them for bundle adjustment. 3. Added bundle adjustment refinement for camera extrinsic calibration. 4. Added `CameraMatrixInitial` field into the XML calibration files to keep the information of the original camera extrinsic parameters when bundle adjustment is run. - 27. Video with the 3D output can be saved with the new `--write_video_3d` flag. - 28. Added Mac OpenCL compatibility. - 29. Added documentation for Nvidia TX2 with JetPack 3.3. - 30. Added Travis build check for several configurations: Ubuntu (14/16)/Mac/Windows and CPU/CUDA/OpenCL and with/without Python. - 31. Assigned 755 access to all sh scripts (some of them were only 644). - 32. Added the flags `--prototxt_path` and `--caffemodel_path` to allow custom ProtoTxt and CaffeModel paths. - 33. Replaced the old Python wrapper for an updated Pybind11 wrapper version, that includes all the functionality of the C++ API. + 27. Added Mac OpenCL compatibility. + 28. Added documentation for Nvidia TX2 with JetPack 3.3. + 29. Added Travis build check for several configurations: Ubuntu (14/16)/Mac/Windows, CPU/CUDA/OpenCL, with/without Python, and Release/Debug. + 30. Assigned 755 access to all sh scripts (some of them were only 644). + 31. Added the flags `--prototxt_path` and `--caffemodel_path` to allow custom ProtoTxt and CaffeModel paths. + 32. Replaced the old Python wrapper for an updated Pybind11 wrapper version, that includes all the functionality of the C++ API. 2. Functions or parameters renamed: 1. By default, python example `tutorial_developer/python_2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368. 2. WrapperStructPose default parameters changed to match those of the OpenPose demo binary. diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index fb02da82..5858e3ab 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -101,8 +101,8 @@ int openPoseDemo() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapper.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_add_module/1_custom_post_processing.cpp b/examples/tutorial_add_module/1_custom_post_processing.cpp index d9bee38e..ccd929dc 100644 --- a/examples/tutorial_add_module/1_custom_post_processing.cpp +++ b/examples/tutorial_add_module/1_custom_post_processing.cpp @@ -114,8 +114,8 @@ int tutorialAddModule1() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp b/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp index 675e194e..a1c78d8a 100644 --- a/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp +++ b/examples/tutorial_api_cpp/3_keypoints_from_image_configurable.cpp @@ -114,8 +114,8 @@ int tutorialApiCpp3() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp b/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp index 085ace96..3183cd24 100644 --- a/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp +++ b/examples/tutorial_api_cpp/4_asynchronous_loop_custom_input_and_output.cpp @@ -249,8 +249,8 @@ int tutorialApiCpp4() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp b/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp index d4d7e1ca..53e5dd43 100644 --- a/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp +++ b/examples/tutorial_api_cpp/5_asynchronous_loop_custom_output.cpp @@ -197,8 +197,8 @@ int tutorialApiCpp5() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp b/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp index 86997667..35b4ce70 100644 --- a/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp +++ b/examples/tutorial_api_cpp/6_synchronous_custom_postprocessing.cpp @@ -165,8 +165,8 @@ int tutorialApiCpp6() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp b/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp index 6cff1cd3..bc70da33 100644 --- a/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp +++ b/examples/tutorial_api_cpp/7_synchronous_custom_input.cpp @@ -195,8 +195,8 @@ int tutorialApiCpp7() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp b/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp index 199a4c74..8d5f0298 100644 --- a/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp +++ b/examples/tutorial_api_cpp/8_synchronous_custom_output.cpp @@ -213,8 +213,8 @@ int tutorialApiCpp8() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp b/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp index 59d3cc8b..bd82a428 100644 --- a/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp +++ b/examples/tutorial_api_cpp/9_synchronous_custom_all.cpp @@ -306,8 +306,8 @@ int tutorialApiCpp9() FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/include/openpose/filestream/imageSaver.hpp b/include/openpose/filestream/imageSaver.hpp index efd8fc14..5e51b3dc 100644 --- a/include/openpose/filestream/imageSaver.hpp +++ b/include/openpose/filestream/imageSaver.hpp @@ -14,6 +14,8 @@ namespace op virtual ~ImageSaver(); + void saveImages(const cv::Mat& cvOutputData, const std::string& fileName) const; + void saveImages(const std::vector& cvOutputDatas, const std::string& fileName) const; private: diff --git a/include/openpose/filestream/videoSaver.hpp b/include/openpose/filestream/videoSaver.hpp index 80d56272..a4cb4416 100644 --- a/include/openpose/filestream/videoSaver.hpp +++ b/include/openpose/filestream/videoSaver.hpp @@ -1,8 +1,6 @@ #ifndef OPENPOSE_FILESTREAM_VIDEO_SAVER_HPP #define OPENPOSE_FILESTREAM_VIDEO_SAVER_HPP -#include // cv::Mat -#include // cv::VideoWriter #include namespace op @@ -10,7 +8,9 @@ namespace op class OP_API VideoSaver { public: - VideoSaver(const std::string& videoSaverPath, const int cvFourcc, const double fps); + VideoSaver( + const std::string& videoSaverPath, const int cvFourcc, const double fps, + const std::string& addAudioFromThisVideo = ""); virtual ~VideoSaver(); @@ -21,13 +21,10 @@ namespace op void write(const std::vector& cvMats); private: - const std::string mVideoSaverPath; - const int mCvFourcc; - const double mFps; - Point mCvSize; - bool mVideoStarted; - cv::VideoWriter mVideoWriter; - unsigned int mNumberImages; + // PIMPL idiom + // http://www.cppsamples.com/common-tasks/pimpl.html + struct ImplVideoSaver; + std::unique_ptr upImpl; DELETE_COPY(VideoSaver); }; diff --git a/include/openpose/flags.hpp b/include/openpose/flags.hpp index 6f7387e6..b16c80b3 100644 --- a/include/openpose/flags.hpp +++ b/include/openpose/flags.hpp @@ -218,11 +218,19 @@ DEFINE_string(write_images_format, "png", "File extension and form " function cv::imwrite for all compatible extensions."); DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the" " final path does not finish in `.avi`. It internally uses cv::VideoWriter. Flag" - " `write_video_fps` controls FPS."); + " `write_video_fps` controls FPS. Alternatively, the video extension can be `.mp4`," + " resulting in a file with a much smaller size and allowing `--write_video_with_audio`." + " However, that would require: 1) Ubuntu or Mac system, 2) FFmpeg library installed" + " (`sudo apt-get install ffmpeg`), 3) the creation temporarily of a folder with the same" + " file path than the final video (without the extension) to storage the intermediate frames" + " that will later be used to generate the final MP4 video."); DEFINE_double(write_video_fps, -1., "Frame rate for the recorded video. By default, it will try to get the input frames producer" " frame rate (e.g., input video or webcam frame rate). If the input frames producer does not" " have a set FPS (e.g., image_dir or webcam if OpenCV not compiled with its support), set" " this value accordingly (e.g., to the frame rate displayed by the OpenPose GUI)."); +DEFINE_bool(write_video_with_audio, false, "If the input is video and the output is so too, it will save the video with audio. It" + " requires the output video file path finishing in `.mp4` format (see `write_video` for" + " details)."); DEFINE_string(write_video_3d, "", "Analogous to `--write_video`, but applied to the 3D output."); DEFINE_string(write_video_adam, "", "Experimental, not available yet. Analogous to `--write_video`, but applied to Adam model."); DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose" diff --git a/include/openpose/utilities/fileSystem.hpp b/include/openpose/utilities/fileSystem.hpp index 1d78b980..c89c8656 100644 --- a/include/openpose/utilities/fileSystem.hpp +++ b/include/openpose/utilities/fileSystem.hpp @@ -35,6 +35,7 @@ namespace op /** * This function extracts the extension from a full path. + * E.g., if fullPath is `/media/document.txt`, output will be `txt` * @param fullPath std::string with the full path. * @return std::string with the file extension. */ diff --git a/include/openpose/wrapper/wrapperAuxiliary.hpp b/include/openpose/wrapper/wrapperAuxiliary.hpp index 9d514709..305469cd 100644 --- a/include/openpose/wrapper/wrapperAuxiliary.hpp +++ b/include/openpose/wrapper/wrapperAuxiliary.hpp @@ -700,12 +700,19 @@ namespace op if (!wrapperStructOutput.writeVideo.empty()) { log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + // Sanity checks if (!oPProducer) error("Video file can only be recorded inside `wrapper/wrapper.hpp` if the producer" " is one of the default ones (e.g., video, webcam, ...).", __LINE__, __FUNCTION__, __FILE__); + if (wrapperStructOutput.writeVideoWithAudio && producerSharedPtr->getType() != ProducerType::Video) + error("Audio can only be added to the output saved video if the input is also a video (either" + " disable `--write_video_with_audio` or use a video as input with `--video`).", + __LINE__, __FUNCTION__, __FILE__); + // Create video saver worker const auto videoSaver = std::make_shared( - wrapperStructOutput.writeVideo, CV_FOURCC('M','J','P','G'), originalVideoFps); + wrapperStructOutput.writeVideo, CV_FOURCC('M','J','P','G'), originalVideoFps, + (wrapperStructOutput.writeVideoWithAudio ? wrapperStructInput.producerString : "")); outputWs.emplace_back(std::make_shared>(videoSaver)); } log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); @@ -800,7 +807,7 @@ namespace op if (!wrapperStructOutput.writeVideo3D.empty()) { const auto videoSaver = std::make_shared( - wrapperStructOutput.writeVideo3D, CV_FOURCC('M','J','P','G'), originalVideoFps); + wrapperStructOutput.writeVideo3D, CV_FOURCC('M','J','P','G'), originalVideoFps, ""); videoSaver3DW = std::make_shared>(videoSaver); } } diff --git a/include/openpose/wrapper/wrapperStructOutput.hpp b/include/openpose/wrapper/wrapperStructOutput.hpp index 1f7f2898..04b2489b 100644 --- a/include/openpose/wrapper/wrapperStructOutput.hpp +++ b/include/openpose/wrapper/wrapperStructOutput.hpp @@ -82,6 +82,20 @@ namespace op */ std::string writeVideo; + /** + * Frame rate of the recorded video. + * By default (-1.), it will try to get the input frames producer frame rate (e.g., input video or webcam frame + * rate). If the input frames producer does not have a set FPS (e.g., image_dir or webcam if OpenCV not + * compiled with its support), set this value accordingly (e.g., to the frame rate displayed by the OpenPose + * GUI). + */ + double writeVideoFps; + + /** + * Whether to save the output video with audio. The input producer must be a video too. + */ + bool writeVideoWithAudio; + /** * Rendered heat maps saving folder. * In order to save the heatmaps, WrapperStructPose.heatMapTypes must also be filled. @@ -95,15 +109,6 @@ namespace op */ std::string writeHeatMapsFormat; - /** - * Frame rate of the recorded video. - * By default (-1.), it will try to get the input frames producer frame rate (e.g., input video or webcam frame - * rate). If the input frames producer does not have a set FPS (e.g., image_dir or webcam if OpenCV not - * compiled with its support), set this value accordingly (e.g., to the frame rate displayed by the OpenPose - * GUI). - */ - double writeVideoFps; - /** * Rendered 3D images saving video path. * Please, use *.avi format. @@ -146,10 +151,11 @@ namespace op const std::string& writeCocoJson = "", const std::string& writeCocoFootJson = "", const int writeCocoJsonVariant = 1, const std::string& writeImages = "", const std::string& writeImagesFormat = "", const std::string& writeVideo = "", - const double writeVideoFps = -1., const std::string& writeHeatMaps = "", - const std::string& writeHeatMapsFormat = "", const std::string& writeVideo3D = "", - const std::string& writeVideoAdam = "", const std::string& writeBvh = "", - const std::string& udpHost = "", const std::string& udpPort = ""); + const double writeVideoFps = -1., const bool writeVideoWithAudio = false, + const std::string& writeHeatMaps = "", const std::string& writeHeatMapsFormat = "", + const std::string& writeVideo3D = "", const std::string& writeVideoAdam = "", + const std::string& writeBvh = "", const std::string& udpHost = "", + const std::string& udpPort = ""); }; } diff --git a/python/openpose/openpose_python.cpp b/python/openpose/openpose_python.cpp index 2e06ee11..ee0c30a7 100644 --- a/python/openpose/openpose_python.cpp +++ b/python/openpose/openpose_python.cpp @@ -85,12 +85,12 @@ public: if (!FLAGS_write_keypoint.empty()) op::log("Flag `write_keypoint` is deprecated and will eventually be removed." " Please, use `write_json` instead.", op::Priority::Max); - // keypointScale - const auto keypointScale = op::flagsToScaleMode(FLAGS_keypoint_scale); + // keypointScaleMode + const auto keypointScaleMode = op::flagsToScaleMode(FLAGS_keypoint_scale); // heatmaps to add const auto heatMapTypes = op::flagsToHeatMaps(FLAGS_heatmaps_add_parts, FLAGS_heatmaps_add_bkg, FLAGS_heatmaps_add_PAFs); - const auto heatMapScale = op::flagsToHeatMapScaleMode(FLAGS_heatmaps_scale); + const auto heatMapScaleMode = op::flagsToHeatMapScaleMode(FLAGS_heatmaps_scale); // >1 camera view? const auto multipleView = (FLAGS_3d || FLAGS_3d_views > 1); // Enabling Google Logging @@ -98,10 +98,10 @@ public: // Pose configuration (use WrapperStructPose{} for default and recommended configuration) const op::WrapperStructPose wrapperStructPose{ - !FLAGS_body_disable, netInputSize, outputSize, keypointScale, FLAGS_num_gpu, FLAGS_num_gpu_start, + !FLAGS_body_disable, netInputSize, outputSize, keypointScaleMode, FLAGS_num_gpu, FLAGS_num_gpu_start, FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, - FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, + FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScaleMode, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, FLAGS_prototxt_path, FLAGS_caffemodel_path, enableGoogleLogging}; opWrapper->configure(wrapperStructPose); @@ -125,8 +125,8 @@ public: FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, - FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, - FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + FLAGS_write_video_with_audio, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, + FLAGS_write_video_adam, FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; opWrapper->configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/scripts/travis/configure_cmake.sh b/scripts/travis/configure_cmake.sh index 6364d41a..59bf34e9 100755 --- a/scripts/travis/configure_cmake.sh +++ b/scripts/travis/configure_cmake.sh @@ -52,6 +52,11 @@ else ARGS="$ARGS -DUSE_CUDNN=Off" fi +echo "WITH_DEBUG = ${WITH_DEBUG}." +if [[ $WITH_DEBUG == true ]] ; then + ARGS="$ARGS -DCMAKE_BUILD_TYPE=Debug" +fi + echo "ARGS = ${ARGS}." cmake .. $ARGS diff --git a/scripts/travis/defaults.sh b/scripts/travis/defaults.sh index e5354f30..5edb4c2b 100755 --- a/scripts/travis/defaults.sh +++ b/scripts/travis/defaults.sh @@ -8,6 +8,7 @@ WITH_CUDA=${WITH_CUDA:-true} WITH_CUDNN=${WITH_CUDNN:-true} WITH_OPEN_CL=${WITH_OPEN_CL:-false} WITH_MKL=${WITH_MKL:-false} +WITH_DEBUG=${WITH_DEBUG:-false} # Examples should be run (Travis not compatible with GPU code) # if [[ $WITH_CMAKE == true ]] && [[ $WITH_PYTHON == true ]] && [[ $WITH_CUDA == false ]] && [[ $WITH_OPEN_CL == false ]] && [[ $WITH_MKL == false ]]; then @@ -19,7 +20,8 @@ fi echo "RUN_EXAMPLES = ${RUN_EXAMPLES}." # Makefile configuration compatible -if [[ $WITH_PYTHON == false ]] ; then +# if [[ $WITH_PYTHON == false ]] ; then +if [[ $WITH_PYTHON == false ]] && [[ $WITH_DEBUG == false ]]; then MAKE_CONFIG_COMPATIBLE=true else MAKE_CONFIG_COMPATIBLE=false diff --git a/src/openpose/filestream/imageSaver.cpp b/src/openpose/filestream/imageSaver.cpp index e0aef1d7..72154185 100644 --- a/src/openpose/filestream/imageSaver.cpp +++ b/src/openpose/filestream/imageSaver.cpp @@ -22,6 +22,18 @@ namespace op { } + void ImageSaver::saveImages(const cv::Mat& cvOutputData, const std::string& fileName) const + { + try + { + saveImages(std::vector{cvOutputData}, fileName); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + void ImageSaver::saveImages(const std::vector& cvOutputDatas, const std::string& fileName) const { try diff --git a/src/openpose/filestream/videoSaver.cpp b/src/openpose/filestream/videoSaver.cpp index d84e5f7b..66100b4f 100644 --- a/src/openpose/filestream/videoSaver.cpp +++ b/src/openpose/filestream/videoSaver.cpp @@ -1,7 +1,49 @@ +#include // cv::VideoWriter +#include +#include +#include #include namespace op { + const auto RANDOM_TEXT = "_r8904530ijyiopf9034jiop4g90j0yh795640h38j"; + + struct VideoSaver::ImplVideoSaver + { + const std::string mVideoSaverPath; + const int mCvFourcc; + const double mFps; + const std::string mAddAudioFromThisVideo; + const bool mUseFfmpeg; + Point mCvSize; + bool mVideoStarted; + unsigned long long mImageSaverCounter; + cv::VideoWriter mVideoWriter; + std::unique_ptr upImageSaver; + std::string mTempImageFolder; + + ImplVideoSaver(const std::string& videoSaverPath, const int cvFourcc, const double fps, + const std::string& addAudioFromThisVideo) : + mVideoSaverPath{videoSaverPath}, + mCvFourcc{cvFourcc}, + mFps{fps}, + mAddAudioFromThisVideo{addAudioFromThisVideo}, + mUseFfmpeg{toLower(getFileExtension(videoSaverPath)) == "mp4"}, + mVideoStarted{false}, + mImageSaverCounter{0ull} + { + try + { + if (mUseFfmpeg) + mTempImageFolder = getFullFilePathNoExtension(mVideoSaverPath) + RANDOM_TEXT; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + }; + cv::VideoWriter openVideo(const std::string& videoSaverPath, const int cvFourcc, const double fps, const Point& cvSize) { @@ -31,17 +73,28 @@ namespace op } } - VideoSaver::VideoSaver(const std::string& videoSaverPath, const int cvFourcc, const double fps) : - mVideoSaverPath{videoSaverPath}, - mCvFourcc{cvFourcc}, - mFps{fps}, - mVideoStarted{false} + VideoSaver::VideoSaver(const std::string& videoSaverPath, const int cvFourcc, const double fps, + const std::string& addAudioFromThisVideo) : + upImpl{new ImplVideoSaver{videoSaverPath, cvFourcc, fps, addAudioFromThisVideo}} { try { - // Sanity check + // Sanity checks if (fps <= 0.) error("Desired fps (frame rate) to save the video is <= 0.", __LINE__, __FUNCTION__, __FILE__); + #ifdef _WIN32 + if (upImpl->mUseFfmpeg) + error("MP4 recording requires an Ubuntu or Mac machine.", __LINE__, __FUNCTION__, __FILE__); + #endif + if (upImpl->mUseFfmpeg && system("ffmpeg --help") != 0) + error("In order to save the video in MP4 format, FFmpeg must be installed on your system." + " Please, use an `avi` output format (e.g., `--write_video output.avi`) or install FFmpeg" + " by running `sudo apt-get install ffmpeg` (Ubuntu) or an analogous command.", + __LINE__, __FUNCTION__, __FILE__); + if (!upImpl->mAddAudioFromThisVideo.empty() && !upImpl->mUseFfmpeg) + error("In order to save the video with audio, it must be in MP4 format. So either 1) do not set" + " `--write_video_audio` or 2) make sure `--write_video` finishes in `.mp4`.", + __LINE__, __FUNCTION__, __FILE__); } catch (const std::exception& e) { @@ -51,13 +104,65 @@ namespace op VideoSaver::~VideoSaver() { + try + { + // Images --> Video + if (upImpl->mUseFfmpeg) + { + log("JPG images temporarily generated in " + upImpl->mTempImageFolder + ".", op::Priority::High); + // FFmpeg command: Save video from images (override if video with same name exists) + const std::string imageToVideoCommand = "ffmpeg -y -i " + upImpl->mTempImageFolder + "/%12d_rendered.jpg" + + " -c:v libx264 -framerate " + std::to_string(upImpl->mFps) + " -pix_fmt yuv420p " + + upImpl->mVideoSaverPath; + log("Creating MP4 video out of JPG images by running:\n" + imageToVideoCommand + "\n", + op::Priority::High); + auto codeAnswer = system(imageToVideoCommand.c_str()); + // Remove temporary images + if (codeAnswer == 0) + { + codeAnswer = system(("rm -rf " + upImpl->mTempImageFolder).c_str()); + log("Video saved and temporary image folder removed.", op::Priority::High); + } + // Sanity check + if (codeAnswer != 0) + log("\nVideo " + upImpl->mVideoSaverPath + " could not be saved (exit code: " + + std::to_string(codeAnswer) + "). Make sure you can manually run the following command" + " (with no errors) from the terminal:\n" + imageToVideoCommand, op::Priority::High); + // Video (no sound) --> Video (with sound) + if (!upImpl->mAddAudioFromThisVideo.empty()) + { + const auto tempOutput = upImpl->mVideoSaverPath + RANDOM_TEXT + ".mp4"; + const auto audioCommand = "ffmpeg -y -i " + upImpl->mVideoSaverPath + + " -i " + upImpl->mAddAudioFromThisVideo + " -codec copy -shortest " + tempOutput; + log("Adding audio to video by running:\n" + audioCommand, op::Priority::High); + auto codeAnswer = system(audioCommand.c_str()); + // Move temp output to real output + if (codeAnswer == 0) + codeAnswer = system(("mv " + tempOutput + " " + upImpl->mVideoSaverPath).c_str()); + // Sanity check + if (codeAnswer != 0) + log("\nVideo " + upImpl->mVideoSaverPath + " could not be saved with audio (exit code: " + + std::to_string(codeAnswer) + "). Make sure you can manually run the following command" + " (with no errors) from the terminal:\n" + audioCommand, op::Priority::High); + } + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } } bool VideoSaver::isOpened() { try { - return mVideoWriter.isOpened(); + // FFmpeg video + if (upImpl->mUseFfmpeg) + return (upImpl->upImageSaver != nullptr); + // OpenCV video + else + return upImpl->mVideoWriter.isOpened(); } catch (const std::exception& e) { @@ -91,12 +196,22 @@ namespace op // Open video (1st frame) // Done here and not in the constructor to handle cases where the resolution is not known (e.g., // reading images or multiple cameras) - if (!mVideoStarted) + if (!upImpl->mVideoStarted) { - mVideoStarted = true; + upImpl->mVideoStarted = true; const auto cvSize = cvMats.at(0).size(); - mCvSize = Point{(int)cvMats.size()*cvSize.width, cvSize.height}; - mVideoWriter = openVideo(mVideoSaverPath, mCvFourcc, mFps, mCvSize); + upImpl->mCvSize = Point{(int)cvMats.size()*cvSize.width, cvSize.height}; + // FFmpeg video + if (upImpl->mUseFfmpeg) + { + log("Temporarily saving video frames as JPG images in: " + upImpl->mTempImageFolder, + op::Priority::High); + upImpl->upImageSaver.reset(new ImageSaver{upImpl->mTempImageFolder, "jpg"}); + } + // OpenCV video + else + upImpl->mVideoWriter = openVideo( + upImpl->mVideoSaverPath, upImpl->mCvFourcc, upImpl->mFps, upImpl->mCvSize); } // Sanity check if (!isOpened()) @@ -108,12 +223,20 @@ namespace op else cvOutputData = cvMats.at(0); // Sanity check - if (mCvSize.x != cvOutputData.cols || mCvSize.y != cvOutputData.rows) + if (upImpl->mCvSize.x != cvOutputData.cols || upImpl->mCvSize.y != cvOutputData.rows) error("You selected to write video (`--write_video`), but the frames to be saved have different" " resolution. You can only save frames with the same resolution.", __LINE__, __FUNCTION__, __FILE__); // Save concatenated image - mVideoWriter.write(cvOutputData); + // FFmpeg video + if (upImpl->mUseFfmpeg) + { + upImpl->upImageSaver->saveImages(cvOutputData, toFixedLengthString(upImpl->mImageSaverCounter, 12u)); + upImpl->mImageSaverCounter++; + } + // OpenCV video + else + upImpl->mVideoWriter.write(cvOutputData); } catch (const std::exception& e) { diff --git a/src/openpose/gui/guiAdam.cpp b/src/openpose/gui/guiAdam.cpp index a1f23158..b52eb5d4 100644 --- a/src/openpose/gui/guiAdam.cpp +++ b/src/openpose/gui/guiAdam.cpp @@ -247,7 +247,7 @@ namespace op const auto originalVideoFps = 30; spImpl->spVideoSaver = std::make_shared( spImpl->mWriteAdamRenderAsVideo, CV_FOURCC('M','J','P','G'), - originalVideoFps + originalVideoFps, "" ); } spImpl->spVideoSaver->write(img); diff --git a/src/openpose/net/bodyPartConnectorBase.cpp b/src/openpose/net/bodyPartConnectorBase.cpp index 71bcbd94..680aa172 100644 --- a/src/openpose/net/bodyPartConnectorBase.cpp +++ b/src/openpose/net/bodyPartConnectorBase.cpp @@ -615,11 +615,10 @@ namespace op } template - void removePeopleBelowThresholds(std::vector& validSubsetIndexes, int& numberPeople, - const std::vector, T>>& peopleVector, - const unsigned int numberBodyParts, const int minSubsetCnt, - const T minSubsetScore, const int maxPeaks, - const bool maximizePositives) + void removePeopleBelowThresholds( + std::vector& validSubsetIndexes, int& numberPeople, + const std::vector, T>>& peopleVector, const unsigned int numberBodyParts, + const int minSubsetCnt, const T minSubsetScore, const int maxPeaks, const bool maximizePositives) { try { @@ -637,11 +636,15 @@ namespace op // same foot usually appears as both left and right keypoints) // Pros: Removed tons of false positives // Cons: Standalone leg will never be recorded - if (!maximizePositives && numberBodyParts == 25) + if (!maximizePositives && (numberBodyParts == 25 || numberBodyParts > 70)) { // No consider foot keypoints for that for (auto i = 19 ; i < 25 ; i++) personCounter -= (peopleVector[index].first.at(i) > 0); + // No consider hand keypoints for that + if (numberBodyParts > 70) + for (auto i = 25 ; i < 65 ; i++) + personCounter -= (peopleVector[index].first.at(i) > 0); } const auto personScore = peopleVector[index].second; if (personCounter >= minSubsetCnt && (personScore/personCounter) >= minSubsetScore) @@ -651,7 +654,7 @@ namespace op if (numberPeople == maxPeaks) break; } - else if ((personCounter < 1 && numberBodyParts != 25) || personCounter < 0) + else if ((personCounter < 1 && numberBodyParts != 25 && numberBodyParts < 70) || personCounter < 0) error("Bad personCounter (" + std::to_string(personCounter) + "). Bug in this" " function if this happens.", __LINE__, __FUNCTION__, __FILE__); } @@ -1126,8 +1129,9 @@ namespace op int numberPeople; std::vector validSubsetIndexes; validSubsetIndexes.reserve(fastMin((size_t)maxPeaks, peopleVector.size())); - removePeopleBelowThresholds(validSubsetIndexes, numberPeople, peopleVector, numberBodyParts, minSubsetCnt, - minSubsetScore, maxPeaks, maximizePositives); + removePeopleBelowThresholds( + validSubsetIndexes, numberPeople, peopleVector, numberBodyParts, minSubsetCnt, minSubsetScore, + maxPeaks, maximizePositives); // Fill and return poseKeypoints peopleVectorToPeopleArray(poseKeypoints, poseScores, scaleFactor, peopleVector, validSubsetIndexes, diff --git a/src/openpose/wrapper/wrapperStructOutput.cpp b/src/openpose/wrapper/wrapperStructOutput.cpp index 7ab0150c..3ca1303e 100644 --- a/src/openpose/wrapper/wrapperStructOutput.cpp +++ b/src/openpose/wrapper/wrapperStructOutput.cpp @@ -6,10 +6,10 @@ namespace op const double verbose_, const std::string& writeKeypoint_, const DataFormat writeKeypointFormat_, const std::string& writeJson_, const std::string& writeCocoJson_, const std::string& writeCocoFootJson_, const int writeCocoJsonVariant_, const std::string& writeImages_, const std::string& writeImagesFormat_, - const std::string& writeVideo_, const double writeVideoFps_, const std::string& writeHeatMaps_, - const std::string& writeHeatMapsFormat_, const std::string& writeVideo3D_, - const std::string& writeVideoAdam_, const std::string& writeBvh_, - const std::string& udpHost_, const std::string& udpPort_) : + const std::string& writeVideo_, const double writeVideoFps_, const bool writeVideoWithAudio_, + const std::string& writeHeatMaps_, const std::string& writeHeatMapsFormat_, const std::string& writeVideo3D_, + const std::string& writeVideoAdam_, const std::string& writeBvh_, const std::string& udpHost_, + const std::string& udpPort_) : verbose{verbose_}, writeKeypoint{writeKeypoint_}, writeKeypointFormat{writeKeypointFormat_}, @@ -20,9 +20,10 @@ namespace op writeImages{writeImages_}, writeImagesFormat{writeImagesFormat_}, writeVideo{writeVideo_}, + writeVideoFps{writeVideoFps_}, + writeVideoWithAudio{writeVideoWithAudio_}, writeHeatMaps{writeHeatMaps_}, writeHeatMapsFormat{writeHeatMapsFormat_}, - writeVideoFps{writeVideoFps_}, writeVideo3D{writeVideo3D_}, writeVideoAdam{writeVideoAdam_}, writeBvh{writeBvh_},