diff --git a/doc/release_notes.md b/doc/release_notes.md index 936e004b..b78c8f2d 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -264,6 +264,7 @@ OpenPose Library - Release Notes 1. Added initial single-person tracker for further speed up or visual smoothing (`--tracking` flag). 2. Greedy body part connector implemented in CUDA: +~30% speed up in Nvidia (CUDA) version with default flags and +~10% in maximum accuracy configuration. In addition, it provides a small 0.2% boost in accuracy (default flags). 3. OpenPose can be built as Unity plugin: Added flag `BUILD_UNITY_SUPPORT` and special Unity code. + 4. If camera is unplugged, OpenPose GUI and command line will display a warning and try to reconnect it. 2. Functions or parameters renamed: 1. By default, python example `2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368. 3. Main bugs fixed: diff --git a/include/openpose/producer/ipCameraReader.hpp b/include/openpose/producer/ipCameraReader.hpp index 2522235b..fda52a51 100644 --- a/include/openpose/producer/ipCameraReader.hpp +++ b/include/openpose/producer/ipCameraReader.hpp @@ -26,6 +26,11 @@ namespace op std::string getNextFrameName(); + inline bool isOpened() const + { + return VideoCaptureReader::isOpened(); + } + inline double get(const int capProperty) { return VideoCaptureReader::get(capProperty); diff --git a/include/openpose/producer/videoCaptureReader.hpp b/include/openpose/producer/videoCaptureReader.hpp index 1dea5bca..812020be 100644 --- a/include/openpose/producer/videoCaptureReader.hpp +++ b/include/openpose/producer/videoCaptureReader.hpp @@ -36,10 +36,7 @@ namespace op virtual std::string getNextFrameName() = 0; - inline bool isOpened() const - { - return mVideoCapture.isOpened(); - } + virtual bool isOpened() const = 0; void release(); @@ -52,6 +49,8 @@ namespace op virtual std::vector getRawFrames() = 0; + void resetWebcam(const int index, const bool throwExceptionIfNoOpened); + private: cv::VideoCapture mVideoCapture; diff --git a/include/openpose/producer/videoReader.hpp b/include/openpose/producer/videoReader.hpp index a8275f9f..fb91be1c 100644 --- a/include/openpose/producer/videoReader.hpp +++ b/include/openpose/producer/videoReader.hpp @@ -34,6 +34,11 @@ namespace op std::string getNextFrameName(); + inline bool isOpened() const + { + return VideoCaptureReader::isOpened(); + } + double get(const int capProperty); void set(const int capProperty, const double value); diff --git a/include/openpose/producer/webcamReader.hpp b/include/openpose/producer/webcamReader.hpp index ac03b53a..51348488 100644 --- a/include/openpose/producer/webcamReader.hpp +++ b/include/openpose/producer/webcamReader.hpp @@ -37,11 +37,14 @@ namespace op std::string getNextFrameName(); + bool isOpened() const; + double get(const int capProperty); void set(const int capProperty, const double value); private: + const int mIndex; double mFps; long long mFrameNameCounter; bool mThreadOpened; @@ -49,6 +52,10 @@ namespace op std::mutex mBufferMutex; std::atomic mCloseThread; std::thread mThread; + // Detect camera unplugged + double mLastNorm; + std::atomic mDisconnectedCounter; + Point mResolution; cv::Mat getRawFrame(); @@ -56,6 +63,8 @@ namespace op void bufferingThread(); + bool reset(); + DELETE_COPY(WebcamReader); }; } diff --git a/src/openpose/producer/producer.cpp b/src/openpose/producer/producer.cpp index c3b242cf..c5ecd04f 100644 --- a/src/openpose/producer/producer.cpp +++ b/src/openpose/producer/producer.cpp @@ -184,9 +184,14 @@ namespace op mNumberEmptyFrames = 0; if (mType != ProducerType::ImageDirectory - && (frame.cols != get(CV_CAP_PROP_FRAME_WIDTH) || frame.rows != get(CV_CAP_PROP_FRAME_HEIGHT))) + && ((frame.cols != get(CV_CAP_PROP_FRAME_WIDTH) && get(CV_CAP_PROP_FRAME_WIDTH) > 0) + || (frame.rows != get(CV_CAP_PROP_FRAME_HEIGHT)) && get(CV_CAP_PROP_FRAME_HEIGHT) > 0)) { - log("Frame size changed. Returning empty frame.", Priority::Max, __LINE__, __FUNCTION__, __FILE__); + log("Frame size changed. Returning empty frame.\nExpected vs. received sizes: " + + std::to_string(get(CV_CAP_PROP_FRAME_WIDTH)) + + "x" + std::to_string(get(CV_CAP_PROP_FRAME_HEIGHT)) + + " vs. " + std::to_string(frame.cols) + "x" + std::to_string(frame.rows), + Priority::Max, __LINE__, __FUNCTION__, __FILE__); frame = cv::Mat(); } } diff --git a/src/openpose/producer/videoCaptureReader.cpp b/src/openpose/producer/videoCaptureReader.cpp index 962ec21c..626eb6ee 100644 --- a/src/openpose/producer/videoCaptureReader.cpp +++ b/src/openpose/producer/videoCaptureReader.cpp @@ -6,14 +6,11 @@ namespace op { VideoCaptureReader::VideoCaptureReader(const int index, const bool throwExceptionIfNoOpened) : - Producer{ProducerType::Webcam}, - mVideoCapture{index} + Producer{ProducerType::Webcam} { try { - // Make sure video capture was opened - if (throwExceptionIfNoOpened && !isOpened()) - error("VideoCapture (webcam) could not be opened.", __LINE__, __FUNCTION__, __FILE__); + resetWebcam(index, throwExceptionIfNoOpened); } catch (const std::exception& e) { @@ -68,6 +65,19 @@ namespace op } } + bool VideoCaptureReader::isOpened() const + { + try + { + return mVideoCapture.isOpened(); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return false; + } + } + cv::Mat VideoCaptureReader::getRawFrame() { try @@ -96,6 +106,22 @@ namespace op } } + void VideoCaptureReader::resetWebcam(const int index, const bool throwExceptionIfNoOpened) + { + try + { + // Open webcam + mVideoCapture = cv::VideoCapture{index}; + // Make sure video capture was opened + if (throwExceptionIfNoOpened && !isOpened()) + error("VideoCapture (webcam) could not be opened.", __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + void VideoCaptureReader::release() { try diff --git a/src/openpose/producer/webcamReader.cpp b/src/openpose/producer/webcamReader.cpp index d84c04a6..4e443ff0 100644 --- a/src/openpose/producer/webcamReader.cpp +++ b/src/openpose/producer/webcamReader.cpp @@ -1,36 +1,43 @@ #include -#include #include +#include +#include namespace op { WebcamReader::WebcamReader(const int webcamIndex, const Point& webcamResolution, const double fps, const bool throwExceptionIfNoOpened) : VideoCaptureReader{webcamIndex, throwExceptionIfNoOpened}, + mIndex{webcamIndex}, mFps{fps}, mFrameNameCounter{-1}, - mThreadOpened{false} + mThreadOpened{std::atomic{false}}, + mResolution{webcamResolution} { try { if (isOpened()) { mFrameNameCounter = 0; - if (webcamResolution != Point{}) + if (mResolution != Point{}) { - set(CV_CAP_PROP_FRAME_WIDTH, webcamResolution.x); - set(CV_CAP_PROP_FRAME_HEIGHT, webcamResolution.y); - if ((int)get(CV_CAP_PROP_FRAME_WIDTH) != webcamResolution.x - || (int)get(CV_CAP_PROP_FRAME_HEIGHT) != webcamResolution.y) + set(CV_CAP_PROP_FRAME_WIDTH, mResolution.x); + set(CV_CAP_PROP_FRAME_HEIGHT, mResolution.y); + if ((int)get(CV_CAP_PROP_FRAME_WIDTH) != mResolution.x + || (int)get(CV_CAP_PROP_FRAME_HEIGHT) != mResolution.y) { - const std::string logMessage{ "Desired webcam resolution " + std::to_string(webcamResolution.x) - + "x" + std::to_string(webcamResolution.y) - + " could not being set. Final resolution: " - + std::to_string(intRound(get(CV_CAP_PROP_FRAME_WIDTH))) + "x" - + std::to_string(intRound(get(CV_CAP_PROP_FRAME_HEIGHT))) }; + const std::string logMessage{ + "Desired webcam resolution " + std::to_string(mResolution.x) + "x" + + std::to_string(mResolution.y) + " could not being set. Final resolution: " + + std::to_string(intRound(get(CV_CAP_PROP_FRAME_WIDTH))) + "x" + + std::to_string(intRound(get(CV_CAP_PROP_FRAME_HEIGHT))) }; log(logMessage, Priority::Max, __LINE__, __FUNCTION__, __FILE__); } } + // Set resolution + mResolution = Point{ + intRound(get(CV_CAP_PROP_FRAME_WIDTH)), + intRound(get(CV_CAP_PROP_FRAME_HEIGHT))}; // Start buffering thread mThreadOpened = true; mThread = std::thread{&WebcamReader::bufferingThread, this}; @@ -111,6 +118,19 @@ namespace op } } + bool WebcamReader::isOpened() const + { + try + { + return (VideoCaptureReader::isOpened() || mDisconnectedCounter > 0); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return false; + } + } + double WebcamReader::get(const int capProperty) { try @@ -194,6 +214,7 @@ namespace op } } + const auto DISCONNETED_THRESHOLD = 15; void WebcamReader::bufferingThread() { try @@ -201,8 +222,29 @@ namespace op mCloseThread = false; while (!mCloseThread) { + // Reset camera if disconnected + bool cameraConnected = true; + if (mDisconnectedCounter > DISCONNETED_THRESHOLD) + cameraConnected = reset(); // Get frame auto cvMat = VideoCaptureReader::getRawFrame(); + // Detect whether camera is connected + const auto newNorm = ( + cvMat.empty() ? mLastNorm : cv::norm(cvMat.row(cvMat.rows/2))); + if (mLastNorm == newNorm) + mDisconnectedCounter++; + else + { + mLastNorm = newNorm; + mDisconnectedCounter = 0; + } + // Camera disconnected: black image + if (!cameraConnected || cvMat.empty()) + { + cvMat = cv::Mat(mResolution.y, mResolution.x, CV_8UC3, cv::Scalar{0,0,0}); + putTextOnCvMat(cvMat, "Camera disconnected, reconnecting...", {cvMat.cols/16, cvMat.rows/2}, + cv::Scalar{255, 255, 255}, false, 2.3*cvMat.cols); + } // Move to buffer if (!cvMat.empty()) { @@ -216,4 +258,33 @@ namespace op error(e.what(), __LINE__, __FUNCTION__, __FILE__); } } + + bool WebcamReader::reset() + { + try + { + // If unplugged + log("Webcam was unplugged, trying to reconnect it.", Priority::Max, + __LINE__, __FUNCTION__, __FUNCTION__); + // Sleep + std::this_thread::sleep_for(std::chrono::milliseconds{1000}); + // Reset camera + VideoCaptureReader::resetWebcam(mIndex, false); + // Re-set resolution + if (isOpened()) + { + set(CV_CAP_PROP_FRAME_WIDTH, mResolution.x); + set(CV_CAP_PROP_FRAME_HEIGHT, mResolution.y); + } + // Camera replugged? + return (!isOpened() + && (mResolution.x != intRound(get(CV_CAP_PROP_FRAME_WIDTH)) + || mResolution.y != intRound(get(CV_CAP_PROP_FRAME_HEIGHT)))); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return false; + } + } }