diff --git a/.gitignore b/.gitignore index 90268bd2..f1aae400 100644 --- a/.gitignore +++ b/.gitignore @@ -94,8 +94,6 @@ Makefile.config 3rdparty/caffe/.git 3rdparty/caffe/.github distribute/ -.build_release/ -.build_debug/ *.caffemodel *.caffemodel.h5 *.solverstate @@ -108,8 +106,8 @@ distribute/ *.bin *.testbin build -*build_debug/ -*build_release/ +build* +.build* *cmake_build distribute/* python/caffe/proto/ diff --git a/CMakeLists.txt b/CMakeLists.txt index be070e30..7e8496b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,9 +151,7 @@ if (${GPU_MODE} MATCHES "CUDA") endif (${GPU_MODE} MATCHES "CUDA") # Suboptions for OpenPose 3D Reconstruction demo -if (WIN32) - option(WITH_3D "Build OpenPose 3D Reconstruction demo." OFF) -endif (WIN32) +option(BUILD_MODULE_3D "Build OpenPose 3D Reconstruction module." OFF) # Download the models option(DOWNLOAD_COCO_MODEL "Download COCO model." ON) @@ -180,6 +178,12 @@ if (${GPU_MODE} MATCHES "CUDA") find_package(CUDA) endif () +# Adding 3D +if (BUILD_MODULE_3D) + # OpenPose flags + add_definitions(-DBUILD_MODULE_3D) +endif (BUILD_MODULE_3D) + if (UNIX AND NOT APPLE) if (${GPU_MODE} MATCHES "CUDA") include(cmake/Cuda.cmake) @@ -189,6 +193,18 @@ if (UNIX AND NOT APPLE) find_package(Glog) find_package(OpenCV) + # 3D + if (BUILD_MODULE_3D) + # GLUT + find_package(GLUT REQUIRED) + # Spinnaker + find_package(Spinnaker) + if (NOT SPINNAKER_FOUND) + message(FATAL_ERROR "Spinnaker not found. Either turn off the `BUILD_MODULE_3D` option or specify the path of + Spinnaker includes and libs.") + endif (NOT SPINNAKER_FOUND) + endif (BUILD_MODULE_3D) + # OpenMP if (${GPU_MODE} MATCHES "CPU_ONLY") find_package(OpenMP) @@ -253,9 +269,9 @@ if (WIN32) download_zip("opencv_310.zip" ${OP_WIN_URL} ${FIND_LIB_PREFIX} 1e5240a64b814b3c0b822f136be78ad7) download_zip("caffe3rdparty_2017_07_14.zip" ${OP_WIN_URL} ${FIND_LIB_PREFIX} ec0f800c8fb337e33304f3375bd06a80) download_zip("caffe_2018_01_18.zip" ${OP_WIN_URL} ${FIND_LIB_PREFIX} 4b8e548cc7ea20abea472950dd5301bd) - if (WITH_3D) + if (BUILD_MODULE_3D) download_zip("freeglut_2018_01_14.zip" ${OP_WIN_URL} ${FIND_LIB_PREFIX} BB182187285E06880F0EDE3A39530091) - endif (WITH_3D) + endif (BUILD_MODULE_3D) message(STATUS "Windows dependencies downloaded.") find_library(OpenCV_LIBS opencv_world310 HINTS ${FIND_LIB_PREFIX}/opencv/x64/vc14/lib) @@ -266,20 +282,20 @@ if (WIN32) find_library(OpenCV_LIBS opencv_world310 HINTS ${FIND_LIB_PREFIX}/opencv/x64/vc14/lib) find_library(Caffe_LIB caffe HINTS ${FIND_LIB_PREFIX}/caffe/lib) find_library(Caffe_Proto_LIB caffeproto HINTS ${FIND_LIB_PREFIX}/caffe/lib) - if (WITH_3D) - find_library(Spinnaker_LIB spinnaker_v140 HINTS ${FIND_LIB_PREFIX}/spinnaker/lib) - find_library(Freeglut_LIB freeglut HINTS ${FIND_LIB_PREFIX}/freeglut/lib) - message(STATUS "\${Freeglut_LIB} = ${Freeglut_LIB}") - endif (WITH_3D) + if (BUILD_MODULE_3D) + find_library(SPINNAKER_LIB spinnaker_v140 HINTS ${FIND_LIB_PREFIX}/spinnaker/lib) + find_library(GLUT_LIBRARY freeglut HINTS ${FIND_LIB_PREFIX}/freeglut/lib) + message(STATUS "\${GLUT_LIBRARY} = ${GLUT_LIBRARY}") + endif (BUILD_MODULE_3D) set(Caffe_LIBS ${Caffe_LIB};${Caffe_Proto_LIB}) set(OpenCV_INCLUDE_DIRS "3rdparty/windows/opencv/include") set(Caffe_INCLUDE_DIRS "3rdparty/windows/caffe/include;3rdparty/windows/caffe/include2") set(Boost_INCLUDE_DIRS "3rdparty/windows/caffe3rdparty/include/boost-1_61") set(WINDOWS_INCLUDE_DIRS "3rdparty/windows/caffe3rdparty/include") - if (WITH_3D) + if (BUILD_MODULE_3D) set(SPINNAKER_INCLUDE_DIRS "3rdparty/windows/spinnaker/include") - set(FREEGLUT_INCLUDE_DIRS "3rdparty/windows/freeglut/include") - endif (WITH_3D) + set(GLUT_INCLUDE_DIRS "3rdparty/windows/freeglut/include") + endif (BUILD_MODULE_3D) endif (WIN32) @@ -400,7 +416,7 @@ if (UNIX AND NOT APPLE) if (NOT Caffe_FOUND AND NOT BUILD_CAFFE) message(FATAL_ERROR "Caffe not found. Either turn on the BUILD_CAFFE option or specify the path of Caffe includes - and libs using -DCaffe_INCLUDE_DIRS and -DCaffe_LIBS") + and libs using -DCaffe_INCLUDE_DIRS and -DCaffe_LIBS.") endif (NOT Caffe_FOUND AND NOT BUILD_CAFFE) endif (${DL_FRAMEWORK} MATCHES "CAFFE") @@ -419,15 +435,15 @@ if (${GPU_MODE} MATCHES "CUDA") include_directories( ${CUDA_INCLUDE_DIRS}) endif () +# 3D +if (BUILD_MODULE_3D) + include_directories(${GLUT_INCLUDE_DIRS} ${SPINNAKER_INCLUDE_DIRS}) +endif (BUILD_MODULE_3D) +# Windows includes if (WIN32) include_directories( ${Boost_INCLUDE_DIRS} ${WINDOWS_INCLUDE_DIRS}) - if (WITH_3D) - include_directories( - ${SPINNAKER_INCLUDE_DIRS} - ${FREEGLUT_INCLUDE_DIRS}) - endif (WITH_3D) endif (WIN32) ### ADD SUBDIRECTORIES @@ -435,9 +451,6 @@ endif (WIN32) add_subdirectory(src) if (BUILD_EXAMPLES) add_subdirectory(examples) - if (WITH_3D) - add_subdirectory(examples_beta) - endif (WITH_3D) endif (BUILD_EXAMPLES) @@ -476,7 +489,7 @@ if (UNIX AND NOT APPLE) COMMENT "Generating API documentation with Doxygen" VERBATIM) else (DOXYGEN_FOUND) - message(FATAL_ERROR "Doxygen need to be installed to generate the doxygen documentation") + message(FATAL_ERROR "Doxygen need to be installed to generate the doxygen documentation.") endif (DOXYGEN_FOUND) endif (BUILD_DOCS) diff --git a/cmake/Modules/FindCaffe.cmake b/cmake/Modules/FindCaffe.cmake index 3736270f..acd1b050 100644 --- a/cmake/Modules/FindCaffe.cmake +++ b/cmake/Modules/FindCaffe.cmake @@ -1,15 +1,15 @@ -# Copied from +# Copied from # https://raw.githubusercontent.com/opencv/opencv_contrib/master/modules/cnn_3dobj/FindCaffe.cmake unset(Caffe_FOUND) unset(Caffe_INCLUDE_DIRS) unset(Caffe_LIBS) -find_path(Caffe_INCLUDE_DIRS NAMES - caffe/caffe.hpp - caffe/common.hpp - caffe/net.hpp - caffe/proto/caffe.pb.h +find_path(Caffe_INCLUDE_DIRS NAMES + caffe/caffe.hpp + caffe/common.hpp + caffe/net.hpp + caffe/proto/caffe.pb.h caffe/util/io.hpp HINTS /usr/local/include) diff --git a/cmake/Modules/FindCuDNN.cmake b/cmake/Modules/FindCuDNN.cmake index 011e5f7c..104eafa8 100644 --- a/cmake/Modules/FindCuDNN.cmake +++ b/cmake/Modules/FindCuDNN.cmake @@ -7,7 +7,7 @@ find_path(CUDNN_INCLUDE cudnn.h get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH) find_library(CUDNN_LIBRARY NAMES ${CUDNN_LIB_NAME} - PATHS ${CUDNN_ROOT} $ENV{CUDNN_ROOT} ${CUDNN_INCLUDE} ${__libpath_hist} ${__libpath_hist}/../lib + PATHS ${CUDNN_ROOT} $ENV{CUDNN_ROOT} ${CUDNN_INCLUDE} ${__libpath_hist} ${__libpath_hist}/../lib DOC "Path to cuDNN library.") if(CUDNN_INCLUDE AND CUDNN_LIBRARY) diff --git a/cmake/Modules/FindSpinnaker.cmake b/cmake/Modules/FindSpinnaker.cmake new file mode 100644 index 00000000..6c5cffbe --- /dev/null +++ b/cmake/Modules/FindSpinnaker.cmake @@ -0,0 +1,20 @@ +# Based on `FindCaffe.cmake` + +unset(SPINNAKER_FOUND) +unset(SPINNAKER_INCLUDE_DIRS) +unset(SPINNAKER_LIB) + +find_path(SPINNAKER_INCLUDE_DIRS NAMES + Spinnaker.h + HINTS + /usr/include/spinnaker/ + /usr/local/include/spinnaker/) + +find_library(SPINNAKER_LIB NAMES Spinnaker + HINTS + /usr/lib + /usr/local/lib) + +if (SPINNAKER_INCLUDE_DIRS AND SPINNAKER_LIB) + set(SPINNAKER_FOUND 1) +endif (SPINNAKER_INCLUDE_DIRS AND SPINNAKER_LIB) diff --git a/cmake/OpenPose.vcxproj.user b/cmake/OpenPose.vcxproj.user index bf1828d4..e3527cbb 100644 --- a/cmake/OpenPose.vcxproj.user +++ b/cmake/OpenPose.vcxproj.user @@ -1,14 +1,14 @@ - + - PATH=%PATH%;3rdparty\windows\caffe\bin\;3rdparty\windows\opencv\x64\vc14\bin;@CMAKE_BINARY_DIR@\src\openpose\Release + PATH=%PATH%;3rdparty\windows\caffe\bin\;3rdparty\windows\opencv\x64\vc14\bin;3rdparty\windows\freeglut\bin\;3rdparty\windows\spinnaker\bin\ WindowsLocalDebugger ..\..\.. - + - PATH=%PATH%;3rdparty\windows\caffe\bin\;3rdparty\windows\opencv\x64\vc14\bin;@CMAKE_BINARY_DIR@\src\openpose\Debug + PATH=%PATH%;3rdparty\windows\caffe\bin\;3rdparty\windows\opencv\x64\vc14\bin;3rdparty\windows\freeglut\bin\;3rdparty\windows\spinnaker\bin\ WindowsLocalDebugger ..\..\.. diff --git a/cmake/OpenPose3D.vcxproj.user b/cmake/OpenPose3D.vcxproj.user deleted file mode 100644 index 1faecf5d..00000000 --- a/cmake/OpenPose3D.vcxproj.user +++ /dev/null @@ -1,16 +0,0 @@ - - - - - PATH=%PATH%;3rdparty\windows\caffe\bin\;3rdparty\windows\freeglut\bin\;3rdparty\windows\spinnaker\bin\;;3rdparty\windows\opencv\x64\vc14\bin;@CMAKE_BINARY_DIR@\src\openpose\Release - WindowsLocalDebugger - ..\..\.. - - - - PATH=%PATH%;3rdparty\windows\caffe\bin\;3rdparty\windows\freeglut\bin\;3rdparty\windows\spinnaker\bin\;;3rdparty\windows\opencv\x64\vc14\bin;@CMAKE_BINARY_DIR@\src\openpose\Debug - WindowsLocalDebugger - ..\..\.. - - - \ No newline at end of file diff --git a/doc/contributors.md b/doc/contributors.md index 0498b6e7..8d879dfc 100644 --- a/doc/contributors.md +++ b/doc/contributors.md @@ -11,6 +11,6 @@ OpenPose is authored by [Gines Hidalgo](https://www.gineshidalgo.com/), [Zhe Cao ### Contributors We would also like to thank the following people who have highly contributed to OpenPose: -1. [Bikramjot Hanzra](https://www.linkedin.com/in/bikz05): OpenPose maintainer and CMake version. +1. [Bikramjot Hanzra](https://www.linkedin.com/in/bikz05): OpenPose maintainer, CMake (Ubuntu and Windows) version, and Travis Build. 2. [Yaadhav Raaj](https://www.linkedin.com/in/yaadhavraaj): OpenPose maintainer, CPU version, and OpenCL version. 3. [Helen Medina](https://github.com/helen-medina): Initial Windows version. diff --git a/doc/installation_cmake.md b/doc/installation_cmake.md index 057c698f..1a7b00ba 100644 --- a/doc/installation_cmake.md +++ b/doc/installation_cmake.md @@ -10,9 +10,9 @@ OpenPose - Installation using CMake 6. [Uninstallation](#uninstallation) 7. [Optional Settings](#optional-settings) 1. [MPI Model](#mpi-model) - 3. [Custom Caffe (Ubuntu Only)](#custom-caffe-ubuntu-only) - 4. [Custom OpenCV (Ubuntu Only)](#custom-opencv-ubuntu-only) - 5. [OpenPose 3D Reconstruction Demo (Windows Only)](#openpose-3d-reconstruction-demo-windows-only) + 3. [OpenPose 3D Reconstruction Module and Demo](#openpose-3d-reconstruction-module-and-demo) + 4. [Custom Caffe (Ubuntu Only)](#custom-caffe-ubuntu-only) + 5. [Custom OpenCV (Ubuntu Only)](#custom-opencv-ubuntu-only) 6. [Doxygen Documentation Autogeneration (Ubuntu Only)](#doxygen-documentation-autogeneration-ubuntu-only) 7. [CMake Command Line Configuration (Ubuntu Only)](#cmake-command-line-configuration-ubuntu-only) @@ -162,6 +162,36 @@ By default, the body MPI model is not downloaded. You can download it by turning +#### OpenPose 3D Reconstruction Module and Demo +You can include the 3D reconstruction module by: + +1. Install the FLIR camera driver and software, Spinnaker SDK. It is a propietary software, so we cannot provide direct download link. + 1. Ubuntu: Get and install the latest Spinnaker SKD version in their default path. OpenPose will automatically find it. + 2. Windows: Donwload the latest Spinnaker SKD version from [https://www.ptgrey.com/support/downloads](https://www.ptgrey.com/support/downloads). + - Copy `{PointGreyParentDirectory}\Point Grey Research\Spinnaker\bin64\vs2015\` as `{OpenPoseDirectory}\3rdparty\windows\spinnaker\bin\`. You can remove all the *.exe files. + - Copy `{PointGreyParentDirectory}\Point Grey Research\Spinnaker\include\` as `{OpenPoseDirectory}\3rdparty\windows\spinnaker\include\`. + - Copy `Spinnaker_v140.lib` and `Spinnakerd_v140.lib` from `{PointGreyParentDirectory}\Point Grey Research\Spinnaker\lib64\vs2015\` into `{OpenPoseDirectory}\3rdparty\windows\spinnaker\lib\`. + - (Optional) Spinnaker SDK overview: [https://www.ptgrey.com/spinnaker-sdk](https://www.ptgrey.com/spinnaker-sdk). +2. Install the 3D visualizer, FreeGLUT: + 1. Ubuntu: run `sudo apt-get install freeglut3 freeglut3-dev`. + 2. Windows: + 1. It is automatically downloaded by the CMake installer. + 2. Alternatively, if you prefer to download it yourself, you could either: + 1. Double click on `3rdparty\windows\getFreeglut.bat`. + 2. Download [this version from our server](http://posefs1.perception.cs.cmu.edu/OpenPose/3rdparty/windows/freeglut_2018_01_14.zip) and unzip it in `{OpenPoseDirectory}\3rdparty\windows\freeglut\`. + 3. Download the latest `MSVC Package` from [http://www.transmissionzero.co.uk/software/freeglut-devel/](http://www.transmissionzero.co.uk/software/freeglut-devel/). + - Copy `{freeglutParentDirectory}\freeglut\bin\x64\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\bin\`. + - Copy `{freeglutParentDirectory}\freeglut\include\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\include\`. + - Copy `{freeglutParentDirectory}\freeglut\lib\x64\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\lib\`. +3. Follow the CMake installation steps, and set the `BUILD_MODULE_3D` option. +4. In Windows, after openning the OpenPose visual studio solution: + 1. Right-click on `Solution 'OpenPose'` of the `Solution Explorer` window, usually placed at the top-right part of the VS screen. + 2. Click on `Properties`. Go to `Configuration Properties` -> `Configuration` and check `Build` for the `OpenPose3DReconstruction` project. + +After installation, check the [doc/openpose_3d_reconstruction_demo.md](./openpose_3d_reconstruction_demo.md) instructions. + + + #### Custom Caffe (Ubuntu Only) We only modified some Caffe compilation flags and minor details. You can use your own Caffe distribution, simply specify the Caffe include path and the library as shown below. You will also need to turn off the `BUILD_CAFFE` variable.

@@ -175,11 +205,6 @@ If you have built OpenCV from source and OpenPose cannot find it automatically, -#### OpenPose 3D Reconstruction Demo (Windows Only) -You can include the 3D Reconstruction Demo by setting the `WITH_3D` option. After installation, check the [doc/openpose_3d_reconstruction_demo.md](./openpose_3d_reconstruction_demo.md) instructions. - - - #### Doxygen Documentation Autogeneration (Ubuntu Only) You can generate the documentation by setting the `BUILD_DOCS` flag. diff --git a/doc/openpose_3d_reconstruction_demo.md b/doc/openpose_3d_reconstruction_demo.md index e53020c8..c2f0204e 100644 --- a/doc/openpose_3d_reconstruction_demo.md +++ b/doc/openpose_3d_reconstruction_demo.md @@ -1,31 +1,29 @@ -Running OpenPose 3-D Reconstruction Demo -==================================== +OpenPose 3-D Reconstruction Module and Demo +============================================= -This is a beta version that makes body + face + hand keypoint 3-D reconstruction and rendering for 1 person. We will not keep updating it nor solving questions/issues about it at the moment. It requires the user to be familiar with computer vision, in particular with camera calibration, i.e. extraction of intrinsic and extrinsic parameters. - -The Windows steps were tested and worked in the OpenPose 1.0.2 version from the last GitHub commit on September 1st, 2017 in the [official repository](https://github.com/CMU-Perceptual-Computing-Lab/openpose). 3D OpenPose should work with any newer version to that one. However, in case it does not, open a GitHub issue and meanwhile use the mentioned version of July 18th. +This experimental module performs 3-D keypoint (body, face, and hand) reconstruction and rendering for 1 person. We will not keep updating it nor solving questions/issues about it at the moment. It requires the user to be familiar with computer vision and camera calibration, including extraction of intrinsic and extrinsic parameters. -### Description of this demo: -- Auto detection of all the FLIR cameras, and extraction of images from each one of them. +## Features +- Auto detection of all FLIR cameras connected to your machine, and image streaming from all of them. - Hardware trigger and buffer `NewestFirstOverwrite` modes enabled. Hence, the algorithm will always get the last synchronized frame from each camera, deleting the rest. -- 3-D reconstruction of body, face and hands for 1 person. +- 3-D reconstruction of body, face, and hands for 1 person. - If more than 1 person is detected per camera, the algorithm will just try to match person 0 on each camera, which will potentially correspond to different people in the scene. Thus, the 3-D reconstruction will completely fail. -- Only points with high threshold with respect to each one of the cameras are reprojected (and later rendered). An alternative for > 4 cameras could be to do 3-D reprojection and render all points with good views in more than N different cameras (not implemented here). -- Only Direct linear transformation (DLT) applied. Non-linear optimization methods (e.g. from Ceres Solver) will potentially improve results (not implemented). +- Only points with high threshold with respect to each one of the cameras are reprojected (and later rendered). An alternative for > 4 cameras could potentially do 3-D reprojection and render all points with good views in more than N different cameras (not implemented here). +- Only Direct linear transformation (DLT) is applied for reconstruction. Non-linear optimization methods (e.g. from Ceres Solver) will potentially improve results (not implemented). - Basic OpenGL rendering with the `freeglut` library. -### Hardware -This demo assumes n arbitrary stereo cameras, FLIR company (former Point Grey). Ideally any USB-3 FLIR model should work, but we have only used the following specific specifications: +## Required Hardware +This demo assumes n arbitrary stereo cameras from the FLIR company (formerly Point Grey). Ideally any USB-3 FLIR model should work, but we have only used the following specific specifications: 1. Camera details: - Blackfly S Color 1.3 MP USB3 Vision (ON Semi PYTHON 1300) - Model: BFS-U3-13Y3C-C - 1280x1024 resolution and 170 FPS - - https://www.ptgrey.com/blackfly-s-13-mp-color-usb3-vision-on-semi-python1300 + - [https://www.ptgrey.com/blackfly-s-13-mp-color-usb3-vision-on-semi-python1300](https://www.ptgrey.com/blackfly-s-13-mp-color-usb3-vision-on-semi-python1300) - Hardware trigger synchronization required. For this camera model, see `Blackfly S` section in [https://www.ptgrey.com/tan/11052](https://www.ptgrey.com/tan/11052) or [https://www.ptgrey.com/KB/11052](https://www.ptgrey.com/KB/11052). - (Ubuntu-only) Open your USB ports following section `Configuring USBFS` in [http://www.ptgrey.com/KB/10685](http://www.ptgrey.com/KB/10685). - Install the Spinnaker SDK for your operating system: [https://www.ptgrey.com/support/downloads](https://www.ptgrey.com/support/downloads). @@ -38,8 +36,8 @@ This demo assumes n arbitrary stereo cameras, FLIR company (former Point Grey). -### Calibrate Cameras -The user must manually get the intrinsic and extrinsic parameters of your cameras and introduce them on: `include\openpose3d\cameraParameters.hpp`. +## Camera Calibration +The user must manually get the intrinsic and extrinsic parameters of the cameras, introduce them on: `include/experimental/3d/cameraParameters.hpp`, and recompile OpenPose. The program uses 3 cameras by default, but cameras can be added or removed from `cameraParameters.hpp` by adding or removing elements to `INTRINSICS`, `DISTORTIONS` and `M_EACH_CAMERA`. `INTRINSICS` corresponds to the intrinsic parameters, `DISTORTIONS` to the distortion coefficients, and `M_EACH_CAMERA` corresponds to the extrinsic parameters of the cameras with respect to camera 1, i.e. camera 1 is considered the coordinates origin. @@ -47,84 +45,30 @@ The program uses 3 cameras by default, but cameras can be added or removed from -### Windows -1. Get the last Spinnaker SKD version, i.e. the FLIR camera driver and software: - - Download last Spinnaker SDK: [https://www.ptgrey.com/support/downloads](https://www.ptgrey.com/support/downloads). - - Copy `{PointGreyParentDirectory}\Point Grey Research\Spinnaker\bin64\vs2015\` as `{OpenPoseDirectory}\3rdparty\windows\spinnaker\bin\`. You can remove all the *.exe files. - - Copy `{PointGreyParentDirectory}\Point Grey Research\Spinnaker\include\` as `{OpenPoseDirectory}\3rdparty\windows\spinnaker\include\`. - - Copy `Spinnaker_v140.lib` and `Spinnakerd_v140.lib` from `{PointGreyParentDirectory}\Point Grey Research\Spinnaker\lib64\vs2015\` into `{OpenPoseDirectory}\3rdparty\windows\spinnaker\lib\`. - - (Optional) Spinnaker SDK overview: [https://www.ptgrey.com/spinnaker-sdk](https://www.ptgrey.com/spinnaker-sdk). -2. Get the last OpenGL Glut library version for the rendering: - 1. CMake installer version: Nothing to download, it is automatically downloaded. - 2. Deprecated Windows installer: - 1. Option a) Double click on `3rdparty\windows\getFreeglut.bat`. - 2. Option b) - - Download the latest `MSVC Package` from [http://www.transmissionzero.co.uk/software/freeglut-devel/](http://www.transmissionzero.co.uk/software/freeglut-devel/). - - Copy `{freeglutParentDirectory}\freeglut\bin\x64\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\bin\`. - - Copy `{freeglutParentDirectory}\freeglut\include\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\include\`. - - Copy `{freeglutParentDirectory}\freeglut\lib\x64\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\lib\`. -3. [Install the OpenPose library](installation.md#installation---library) and make sure the [OpenPose demo runs](quick_start.md#quick-start), so that the required OpenPose library was properly compiled. If you are using CMake installer, enable the `WITH_3D` option. -4. Open the OpenPose visual studio solution: - 1. CMake installer version: `build\openpose.sln`. - 2. Deprecated Windows installer: `windows\openpose.sln`. -5. Right-click on `Solution 'OpenPose'` of the `Solution Explorer` window, usually placed at the top-right part of the VS screen. -6. Click on `Properties`. Go to `Configuration Properties` -> `Configuration` and check `Build` for the `OpenPose3DReconstruction` project. +## Camera Ordering +In order to verify that the camera parameters introduced by the user are sorted in the same way that OpenPose reads the cameras, make sure of the following points: + +1. Initially, introduce the camera parameters sorted by serial number. By default (in Spinnaker 1.8), they are sorted by serial number. +2. When the program is run, OpenPose displays the camera serial number associated to each index of each detected camera. If the number of cameras detected is different to the number of actual cameras, make sure the hardware is properly connected and the camera leds are on. +3. Make sure that the order in which you introduced your camera parameters matches this index ordering displayed by OpenPose. Again, it should be sorted by serial number, but different Spinnaker versions might work differently. -### Ubuntu -We did not create an Ubuntu version. We did an very first version for Ubuntu 16 long ago, but it was highly changed later. These are the steps we used for that one. Note that there might be needed some changes to make it work. Feel free to send us or make a pull request with any updated steps. -1. Install the OpenGL rendering library: `sudo apt-get install freeglut3-dev`. -2. Compile the standard OpenPose [from https://github.com/CMU-Perceptual-Computing-Lab/openpose](from https://github.com/CMU-Perceptual-Computing-Lab/openpose). -3. Perform `make distribute` on OpenPose, and copy the `include` and `lib` files from `distribute` into your custom `3rdparty/openpose/`. -4. Copy the `include` and `lib` folders from {OpenPose path}/3rdparty/caffe/distribute/ into your custom `3rdparty/caffe/`. -5. Copy your Spinnaker desired version `include` and `lib` folders in your custom `3rdparty/spinnaker/`. -7. From the Spinnaker `bin` folder, copy all the *.xml files to the generated build folder of your project. -8. Get the required files from `{OpenPose path}/examples_beta/openpose3d/`. Check the Windows VS solution for more details. -9. Create a proper Makefile or CMake file to run it. The following code is part of an old QMake (Qt) file generated for the old version, you can ideally get all the flags and includes from it: -``` -INCLUDEPATH += \ - $$PWD/include \ - $$PWD/3rdparty/caffe/include \ - $$PWD/3rdparty/openpose/include \ - $$PWD/3rdparty/spinnaker/include \ - /usr/include \ - /usr/local/include \ - /usr/local/cuda-8.0/include -} -# Generic -LIBS += -L/usr/lib/ -L/usr/local/lib/ -L/usr/lib/x86_64-linux-gnu -# OpenPose -LIBS += -Wl,-rpath=$$PWD/3rdparty/openpose/lib -LIBS += -Wl,-Bdynamic -L$$PWD/3rdparty/openpose/lib/ -lopenpose -# Caffe -LIBS += -Wl,-rpath=$$PWD/3rdparty/caffe/lib -LIBS += -Wl,-Bdynamic -L$$PWD/3rdparty/caffe/lib/ -lcaffe -# Spinnaker -LIBS += -Wl,-rpath=$$PWD/3rdparty/spinnaker/lib -LIBS += -Wl,-Bdynamic -L$$PWD/3rdparty/spinnaker/lib/ -lSpinnaker -# OpenCV -LIBS += -lopencv_core -lopencv_highgui -lopencv_imgproc -lopencv_contrib -lopencv_calib3d -# CUDA -LIBS += -I/usr/local/cuda-8.0/include/ -LIBS += -L/usr/local/cuda-8.0/lib64 -lcudart -lcublas -lcurand -# OpenGL -LIBS += -lGLU -lGL -lglut -# Other 3rdparty -LIBS += -lcudnn -lglog -lgflags -lm -LIBS += -pthread -fPIC -std=c++11 -fopenmp -# Optimization flags -LIBS += -DNDEBUG -O3 -march=native -# Debug flags -LIBS += -Wpedantic -Wall -Wextra -Wfatal-errors -``` -10. If you find any error/difference, feel free to add a pull request to help other users. +## Installing the OpenPose 3-D Reconstruction Module +Check the [doc/installation_cmake.md#openpose-3d-reconstruction-module-and-demo](./installation_cmake.md#openpose-3d-reconstruction-module-and-demo) instructions. ## Expected Visual Results -The visual GUI should show 3 screens, the Windows command line or Ubuntu bash terminal, the different cameras 2-D keypoint estimations, and the final 3-D reconstruction, similarly to the following image: +The visual GUI should show 3 screens. + +1. The Windows command line or Ubuntu bash terminal. +2. The different cameras 2-D keypoint estimations. +3. The final 3-D reconstruction. + +It should be similar to the following image. +

diff --git a/doc/release_notes.md b/doc/release_notes.md index 3cccd2f9..e97eabe1 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -179,6 +179,7 @@ OpenPose Library - Release Notes 7. Runtime verbose about average speed configurable by user with `PROFILER_ENABLED` option (CMake/Makefile.config) and `--profile_speed` flag. 8. Lighter Caffe version compiled by CMake in Ubuntu: disabled Caffe extra support (e.g., OpenCV, Python) and doc. 9. Renamed CMake binaries (Ubuntu) to match old Makefile format: `_bin` by `.bin`. + 10. 3-D Reconstruction demo cleaned, implemented in Ubuntu too, and now defined as experimental module of OpenPose rather than just a demo. 2. Main bugs fixed: 1. Slight speed up (~1%) for performing the non-maximum suppression stage only in the body part heatmaps channels, and not also in the PAF channels. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index f72279f3..1f19c3db 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -3,6 +3,9 @@ add_subdirectory(tutorial_add_module) add_subdirectory(tutorial_pose) add_subdirectory(tutorial_thread) add_subdirectory(tutorial_wrapper) +if (BUILD_MODULE_3D) + add_subdirectory(experimental_3d) +endif (BUILD_MODULE_3D) if (UNIX AND NOT APPLE) add_subdirectory(tests) -endif (UNIX AND NOT APPLE) \ No newline at end of file +endif (UNIX AND NOT APPLE) diff --git a/examples_beta/openpose3d/openpose3d.cpp b/examples/experimental_3d/3d.cpp similarity index 98% rename from examples_beta/openpose3d/openpose3d.cpp rename to examples/experimental_3d/3d.cpp index 5510b6f0..1259983b 100644 --- a/examples_beta/openpose3d/openpose3d.cpp +++ b/examples/experimental_3d/3d.cpp @@ -15,9 +15,7 @@ // C++ std library dependencies #include // `std::chrono::` functions and classes, e.g. std::chrono::milliseconds -#include #include // std::this_thread -#include // Other 3rdparty dependencies // GFlags: DEFINE_bool, _int32, _int64, _uint64, _double, _string #include @@ -28,7 +26,6 @@ // OpenPose dependencies #include -#include // See all the available parameter options withe the `--help` flag. E.g. `build/examples/openpose/openpose.bin --help` // Note: This command will show you flags for other unnecessary 3rdparty files. Check only the flags for the OpenPose @@ -169,7 +166,7 @@ DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write DEFINE_string(write_keypoint_json, "", "(Deprecated, use `write_json`) Directory to write people pose data in JSON format," " compatible with any OpenCV version."); -int openpose3d() +int openpose() { // logging_level op::check(0 <= FLAGS_logging_level && FLAGS_logging_level <= 255, "Wrong logging_level value.", @@ -209,13 +206,13 @@ int openpose3d() // Initializing the user custom classes // Frames producer (e.g. video, webcam, ...) - auto wPointGrey = std::make_shared(); + auto wPointGrey = std::make_shared(); // Processing - auto wReconstruction3D = std::make_shared(); + auto wReconstruction3D = std::make_shared(); // GUI (Display) - auto wRender3D = std::make_shared(); + auto wRender3D = std::make_shared(); - op::Wrapper> opWrapper; + op::Wrapper> opWrapper; // Add custom input const auto workerInputOnNewThread = true; opWrapper.setWorkerInput(wPointGrey, workerInputOnNewThread); @@ -280,6 +277,6 @@ int main(int argc, char *argv[]) // Parsing command line flags gflags::ParseCommandLineFlags(&argc, &argv, true); - // Running openpose3d - return openpose3d(); + // Running openpose + return openpose(); } diff --git a/examples/experimental_3d/CMakeLists.txt b/examples/experimental_3d/CMakeLists.txt new file mode 100644 index 00000000..849b62c1 --- /dev/null +++ b/examples/experimental_3d/CMakeLists.txt @@ -0,0 +1,27 @@ +set(EXAMPLE_FILES + 3d.cpp) + +foreach(EXAMPLE_FILE ${EXAMPLE_FILES}) + + get_filename_component(SOURCE_NAME ${EXAMPLE_FILE} NAME_WE) + + if (UNIX AND NOT APPLE) + set(EXE_NAME "${SOURCE_NAME}.bin") + elseif (WIN32) + set(EXE_NAME "OpenPoseDemo3D") + endif () + + message(STATUS "Adding Example ${EXE_NAME}") + add_executable(${EXE_NAME} ${EXAMPLE_FILE}) + target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB}) + + if (WIN32) + set_property(TARGET ${EXE_NAME} PROPERTY FOLDER "Examples") + configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user + ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + # Properties->General->Output Directory + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + endif (WIN32) + +endforeach() diff --git a/examples/openpose/CMakeLists.txt b/examples/openpose/CMakeLists.txt index 905630f9..25bf7640 100644 --- a/examples/openpose/CMakeLists.txt +++ b/examples/openpose/CMakeLists.txt @@ -13,12 +13,15 @@ foreach(EXAMPLE_FILE ${EXAMPLE_FILES}) message(STATUS "Adding Example ${EXE_NAME}") add_executable(${EXE_NAME} ${EXAMPLE_FILE}) - target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS}) + target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB}) if (WIN32) set_property(TARGET ${EXE_NAME} PROPERTY FOLDER "Examples") - configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user - ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user + ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + # Properties->General->Output Directory + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) endif (WIN32) endforeach() diff --git a/examples/tests/CMakeLists.txt b/examples/tests/CMakeLists.txt index f1f67edd..1c743056 100644 --- a/examples/tests/CMakeLists.txt +++ b/examples/tests/CMakeLists.txt @@ -13,12 +13,15 @@ foreach(EXAMPLE_FILE ${EXAMPLE_FILES}) message(STATUS "Adding Example ${EXE_NAME}") add_executable(${EXE_NAME} ${EXAMPLE_FILE}) - target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS}) + target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB}) if (WIN32) set_property(TARGET ${EXE_NAME} PROPERTY FOLDER "Examples/Tutorial/Tests") - configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user - ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user + ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + # Properties->General->Output Directory + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) endif (WIN32) endforeach() diff --git a/examples/tutorial_add_module/CMakeLists.txt b/examples/tutorial_add_module/CMakeLists.txt index 3fb74a35..eabac27f 100644 --- a/examples/tutorial_add_module/CMakeLists.txt +++ b/examples/tutorial_add_module/CMakeLists.txt @@ -16,12 +16,15 @@ foreach(EXAMPLE_FILE ${EXAMPLE_FILES}) message(STATUS "Adding Example ${EXE_NAME}") add_executable(${EXE_NAME} ${EXAMPLE_FILE}) target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} - ${Caffe_LIBS} ${MKL_LIBS}) + ${Caffe_LIBS} ${MKL_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB}) if (WIN32) set_property(TARGET ${EXE_NAME} PROPERTY FOLDER "Examples/Tutorial/AddModule") configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + # Properties->General->Output Directory + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) endif (WIN32) endforeach() diff --git a/examples/tutorial_pose/CMakeLists.txt b/examples/tutorial_pose/CMakeLists.txt index f5cf0a66..82fd00c8 100644 --- a/examples/tutorial_pose/CMakeLists.txt +++ b/examples/tutorial_pose/CMakeLists.txt @@ -14,12 +14,15 @@ foreach(EXAMPLE_FILE ${EXAMPLE_FILES}) message(STATUS "Adding Example ${EXE_NAME}") add_executable(${EXE_NAME} ${EXAMPLE_FILE}) - target_link_libraries(${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS}) + target_link_libraries(${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB}) if (WIN32) set_property(TARGET ${EXE_NAME} PROPERTY FOLDER "Examples/Tutorial/Pose") - configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user - ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user + ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + # Properties->General->Output Directory + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) endif (WIN32) endforeach() diff --git a/examples/tutorial_thread/CMakeLists.txt b/examples/tutorial_thread/CMakeLists.txt index e6a62070..31867e8c 100644 --- a/examples/tutorial_thread/CMakeLists.txt +++ b/examples/tutorial_thread/CMakeLists.txt @@ -16,12 +16,15 @@ foreach(EXAMPLE_FILE ${EXAMPLE_FILES}) message(STATUS "Adding Example ${EXE_NAME}") add_executable(${EXE_NAME} ${EXAMPLE_FILE}) - target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS}) + target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB}) if (WIN32) set_property(TARGET ${EXE_NAME} PROPERTY FOLDER "Examples/Tutorial/Thread") - configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user - ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user + ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + # Properties->General->Output Directory + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) endif (WIN32) endforeach() diff --git a/examples/tutorial_wrapper/CMakeLists.txt b/examples/tutorial_wrapper/CMakeLists.txt index 9c1e1041..85bc8c10 100644 --- a/examples/tutorial_wrapper/CMakeLists.txt +++ b/examples/tutorial_wrapper/CMakeLists.txt @@ -17,12 +17,15 @@ foreach(EXAMPLE_FILE ${EXAMPLE_FILES}) message(STATUS "Adding Example ${EXE_NAME}") add_executable(${EXE_NAME} ${EXAMPLE_FILE}) - target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS}) + target_link_libraries( ${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} ${MKL_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB}) if (WIN32) set_property(TARGET ${EXE_NAME} PROPERTY FOLDER "Examples/Tutorial/Wrapper") - configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user - ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose.vcxproj.user + ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) + # Properties->General->Output Directory + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + set_property(TARGET ${EXE_NAME} PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) endif (WIN32) endforeach() diff --git a/examples_beta/CMakeLists.txt b/examples_beta/CMakeLists.txt deleted file mode 100644 index fd59fcb6..00000000 --- a/examples_beta/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(openpose3d) \ No newline at end of file diff --git a/examples_beta/openpose3d/CMakeLists.txt b/examples_beta/openpose3d/CMakeLists.txt deleted file mode 100644 index a096fba5..00000000 --- a/examples_beta/openpose3d/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -if (WIN32) - - set(OPENPOSE_3D_INCLUDES include) - file(GLOB_RECURSE OPENPOSE_3D_SOURCES "*.cpp") - - set(EXAMPLE_FILES openpose3d.cpp) - - include(${CMAKE_SOURCE_DIR}/cmake/Utils.cmake) - - foreach (EXAMPLE_FILE ${EXAMPLE_FILES}) - - get_filename_component(SOURCE_NAME ${EXAMPLE_FILE} NAME_WE) - - set(EXE_NAME "${SOURCE_NAME}") - - message(STATUS "Adding Example ${EXE_NAME}") - add_executable(${EXE_NAME} ${EXAMPLE_FILE} ${OPENPOSE_3D_SOURCES}) - target_include_directories(${EXE_NAME} PUBLIC ${OPENPOSE_3D_INCLUDES}) - target_link_libraries(${EXE_NAME} openpose ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${Caffe_LIBS} - ${Freeglut_LIB} ${Spinnaker_LIB}) - - set_property(TARGET ${EXE_NAME} PROPERTY FOLDER "Examples/Tutorial/OpenPose3DReconstruction") - configure_file(${CMAKE_SOURCE_DIR}/cmake/OpenPose3D.vcxproj.user - ${CMAKE_CURRENT_BINARY_DIR}/${EXE_NAME}.vcxproj.user @ONLY) - - endforeach () - -endif (WIN32) \ No newline at end of file diff --git a/examples_beta/openpose3d/include/openpose3d/cameraParameters.hpp b/examples_beta/openpose3d/include/openpose3d/cameraParameters.hpp deleted file mode 100644 index 40ae64cb..00000000 --- a/examples_beta/openpose3d/include/openpose3d/cameraParameters.hpp +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef OPENPOSE3D_CAMERA_PARAMETERS_HPP -#define OPENPOSE3D_CAMERA_PARAMETERS_HPP - -#include - -// User configurable code -// Intrinsic and distortion parameters -// Camera 1 parameters -const cv::Mat INTRINSIC_1 = (cv::Mat_(3, 3) << 817.93481631740565, 0, 600.70689997785121, - 0, 816.51774059837908, 517.84529566329593, - 0, 0, 1); -const cv::Mat DISTORTION_1 = (cv::Mat_(8, 1) << - -1.8102158829399091, 9.1966147162623262, -0.00044293900343777355, 0.0013638377686816653, 1.3303863414979364, -1.418905163635487, 8.4725535468475819, 4.7911023525901033); -const cv::Mat INTRINSIC_2 = (cv::Mat_(3, 3) << 816.20921132436638, 0, 612.67087968681585, - 0, 816.18292222910486, 530.47901782670431, - 0, 0, 1); -// Camera 2 parameters -const cv::Mat DISTORTION_2 = (cv::Mat_(8, 1) << - -5.1088507540294881, 133.63995617304997, -0.0010048069080912836, 0.00018825291386406282, 20.688286893903879, -4.7604289550474768, 132.42412342224557, 70.01195364029752); -const cv::Mat INTRINSIC_3 = (cv::Mat_(3, 3) << 798.42980806905666, 0, 646.48130011561727, - 0, 798.46535448393979, 523.91590563194586, - 0, 0, 1); -// Camera 3 -const cv::Mat DISTORTION_3 = (cv::Mat_(8, 1) << - -0.57530495294002304, -0.54721992620722582, -0.00037614702677289967, -0.00081995658363481598, -0.020321660897680775, -0.18040544059116842, -0.87724444571603022, -0.13136636671099691); - -// Extrinsic parameters - rotation and pose orientation between cameras -// From camera 1 to 2 -const cv::Mat M_1_2 = (cv::Mat_(3, 4) << 0.999962504862692, -0.00165862051503619, 0.00849928507093793, -238.301309354482, - 0.00176155163779584, 0.999925029704659, -0.0121174215889211, 4.75863886121558, - -0.00847854967298925, 0.0121319391740716, 0.999890459124058, 15.9219925821916); -// From camera 1 to 3 -const cv::Mat M_1_3 = (cv::Mat_(3, 4) << 0.995809442124071, -0.000473104796892308, 0.0914512501193800, -461.301274485705, - 0.00165046455210419, 0.999916727562850, -0.0127989806923977, 6.22648121362088, - -0.0914375794917412, 0.0128962828696210, 0.995727299487585, 63.4911132860733); -// From camera 2 to 3 -const cv::Mat M_2_3 = (cv::Mat_(3, 4) << 0.999644115423621, -0.00194501088674130, -0.0266056278177532, -235.236375502202, - 0.00201646110733780, 0.999994431880356, 0.00265896462686206, 9.52238656728889, - 0.0266003079592876, -0.00271166755609303, 0.999642471324391, -4.23534963077479); - - - -// Do not modify this code -const std::vector INTRINSICS{ INTRINSIC_1, INTRINSIC_2, INTRINSIC_3 }; -const std::vector DISTORTIONS{ DISTORTION_1, DISTORTION_2, DISTORTION_3 }; -const std::vector M_EACH_CAMERA{ INTRINSIC_1 * cv::Mat::eye(3,4, CV_64F), INTRINSIC_2 * M_1_2, INTRINSIC_3 * M_1_3 }; - -#endif // OPENPOSE3D_CAMERA_PARAMETERS_HPP diff --git a/examples_beta/openpose3d/include/openpose3d/datum3D.hpp b/examples_beta/openpose3d/include/openpose3d/datum3D.hpp deleted file mode 100644 index e389b44d..00000000 --- a/examples_beta/openpose3d/include/openpose3d/datum3D.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef OPENPOSE3D_DATUM_3D_HPP -#define OPENPOSE3D_DATUM_3D_HPP - -#include -#include - -// Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from Datum -// See `examples/tutorial_wrapper/` for more details -struct Datum3D : public op::Datum -{ - op::Array poseKeypoints3D; - op::Array faceKeypoints3D; - op::Array leftHandKeypoints3D; - op::Array rightHandKeypoints3D; -}; - -#endif // OPENPOSE3D_DATUM_3D_HPP diff --git a/examples_beta/openpose3d/include/openpose3d/headers.hpp b/examples_beta/openpose3d/include/openpose3d/headers.hpp deleted file mode 100644 index e201ef9f..00000000 --- a/examples_beta/openpose3d/include/openpose3d/headers.hpp +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef OPENPOSE3D_HEADERS_HPP -#define OPENPOSE3D_HEADERS_HPP - -// Camera intrinsic and extrinsic parameters -#include -// Datum3D -#include -// PointGrey cameras -#include -// 3D reconstruction -#include -// OpenGL Renderer -#include - -#endif // OPENPOSE3D_HEADERS_HPP diff --git a/examples_beta/openpose3d/include/openpose3d/pointGrey.hpp b/examples_beta/openpose3d/include/openpose3d/pointGrey.hpp deleted file mode 100644 index cd1916c8..00000000 --- a/examples_beta/openpose3d/include/openpose3d/pointGrey.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef OPENPOSE3D_POINT_GREY_HPP -#define OPENPOSE3D_POINT_GREY_HPP - -#include -#include -#include - -// Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from WorkerProducer. -// This worker: -// 1. Set hardware trigger and the buffer to get the latest obtained frame. -// 2. Read images from FLIR cameras. -// 3. Turn them into std::vector. -// 4. Return the resulting images wrapped into a std::shared_ptr>. -// The HW trigger + reading FLIR camera code is highly based on the Spinnaker SDK examples `AcquisitionMultipleCamera` and specially `Trigger` -// (located in `src/`). See them for more details about the cameras. -// See `examples/tutorial_wrapper/` for more details about inhering the WorkerProducer class. -class WPointGrey : public op::WorkerProducer>> -{ -public: - WPointGrey(); - - ~WPointGrey(); - - void initializationOnThread(); - - std::shared_ptr> workProducer(); - -private: - bool initialized; - Spinnaker::CameraList mCameraList; - Spinnaker::SystemPtr mSystemPtr; -}; - -#endif // OPENPOSE3D_POINT_GREY_HPP diff --git a/examples_beta/openpose3d/include/openpose3d/reconstruction3D.hpp b/examples_beta/openpose3d/include/openpose3d/reconstruction3D.hpp deleted file mode 100644 index c3a4380b..00000000 --- a/examples_beta/openpose3d/include/openpose3d/reconstruction3D.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef RECONSTRUCTION_3D_HPP -#define RECONSTRUCTION_3D_HPP - -#include - -// Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from Worker. -// This worker will do 3-D reconstruction -// We apply the simple Direct linear transformation (DLT) algorithm, asumming each keypoint (e.g. right hip) is seen by all the cameras. -// No non-linear minimization used, and if some camera misses the point, it is not reconstructed. -// See `examples/tutorial_wrapper/` for more details about inhering the Worker class and using it for post-processing purposes. -class WReconstruction3D : public op::Worker>> -{ -public: - void initializationOnThread() {} - - void work(std::shared_ptr>& datumsPtr); -}; - -#endif // RECONSTRUCTION_3D_HPP diff --git a/examples_beta/openpose3d/include/openpose3d/renderer.hpp b/examples_beta/openpose3d/include/openpose3d/renderer.hpp deleted file mode 100644 index 73b7a1b7..00000000 --- a/examples_beta/openpose3d/include/openpose3d/renderer.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef OPENPOSE3D_RENDERER_HPP -#define OPENPOSE3D_RENDERER_HPP - -#include -#include -#include -#include - -// This worker will do 3-D rendering -class WRender3D : public op::WorkerConsumer>> -{ -public: - WRender3D(const op::PoseModel poseModel = op::PoseModel::COCO_18); - - void initializationOnThread() {} - - void workConsumer(const std::shared_ptr>& datumsPtr); - -private: - std::thread mRenderThread; - - void visualizationThread(); -}; - -#endif // OPENPOSE3D_RENDERER_HPP diff --git a/examples_beta/openpose3d/src/pointGrey.cpp b/examples_beta/openpose3d/src/pointGrey.cpp deleted file mode 100644 index 0431be18..00000000 --- a/examples_beta/openpose3d/src/pointGrey.cpp +++ /dev/null @@ -1,564 +0,0 @@ -#include -#include -#include -#include -#include -#include - -/* - * This function converts between Spinnaker::ImagePtr container to cv::Mat container used in OpenCV. -*/ -cv::Mat pointGreyToCvMat(const Spinnaker::ImagePtr &imagePtr) -{ - try - { - const auto XPadding = imagePtr->GetXPadding(); - const auto YPadding = imagePtr->GetYPadding(); - const auto rowsize = imagePtr->GetWidth(); - const auto colsize = imagePtr->GetHeight(); - - // image data contains padding. When allocating cv::Mat container size, you need to account for the X,Y image data padding. - return cv::Mat((int)(colsize + YPadding), (int)(rowsize + XPadding), CV_8UC3, imagePtr->GetData(), imagePtr->GetStride()); - } - catch (const std::exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return cv::Mat(); - } -} - -// This function configures the camera to use a trigger. First, trigger mode is -// set to off in order to select the trigger source. Once the trigger source -// has been selected, trigger mode is then enabled, which has the camera -// capture only a single image upon the execution of the chosen trigger. -int configureTrigger(Spinnaker::GenApi::INodeMap &iNodeMap) -{ - try - { - int result = 0; - op::log("*** CONFIGURING TRIGGER ***", op::Priority::High); - op::log("Configuring hardware trigger...", op::Priority::High); - // Ensure trigger mode off - // *** NOTES *** - // The trigger must be disabled in order to configure whether the source - // is software or hardware. - Spinnaker::GenApi::CEnumerationPtr ptrTriggerMode = iNodeMap.GetNode("TriggerMode"); - if (!Spinnaker::GenApi::IsAvailable(ptrTriggerMode) || !Spinnaker::GenApi::IsReadable(ptrTriggerMode)) - op::error("Unable to disable trigger mode (node retrieval). Aborting...", __LINE__, __FUNCTION__, __FILE__); - - Spinnaker::GenApi::CEnumEntryPtr ptrTriggerModeOff = ptrTriggerMode->GetEntryByName("Off"); - if (!Spinnaker::GenApi::IsAvailable(ptrTriggerModeOff) || !Spinnaker::GenApi::IsReadable(ptrTriggerModeOff)) - op::error("Unable to disable trigger mode (enum entry retrieval). Aborting...", __LINE__, __FUNCTION__, __FILE__); - - ptrTriggerMode->SetIntValue(ptrTriggerModeOff->GetValue()); - - op::log("Trigger mode disabled...", op::Priority::High); - - // Select trigger source - // *** NOTES *** - // The trigger source must be set to hardware or software while trigger - // mode is off. - Spinnaker::GenApi::CEnumerationPtr ptrTriggerSource = iNodeMap.GetNode("TriggerSource"); - if (!Spinnaker::GenApi::IsAvailable(ptrTriggerSource) || !Spinnaker::GenApi::IsWritable(ptrTriggerSource)) - op::error("Unable to set trigger mode (node retrieval). Aborting...", __LINE__, __FUNCTION__, __FILE__); - - // Set trigger mode to hardware ('Line0') - Spinnaker::GenApi::CEnumEntryPtr ptrTriggerSourceHardware = ptrTriggerSource->GetEntryByName("Line0"); - if (!Spinnaker::GenApi::IsAvailable(ptrTriggerSourceHardware) || !Spinnaker::GenApi::IsReadable(ptrTriggerSourceHardware)) - op::error("Unable to set trigger mode (enum entry retrieval). Aborting...", __LINE__, __FUNCTION__, __FILE__); - - ptrTriggerSource->SetIntValue(ptrTriggerSourceHardware->GetValue()); - - op::log("Trigger source set to hardware...", op::Priority::High); - - // Turn trigger mode on - // *** LATER *** - // Once the appropriate trigger source has been set, turn trigger mode - // on in order to retrieve images using the trigger. - Spinnaker::GenApi::CEnumEntryPtr ptrTriggerModeOn = ptrTriggerMode->GetEntryByName("On"); - if (!Spinnaker::GenApi::IsAvailable(ptrTriggerModeOn) || !Spinnaker::GenApi::IsReadable(ptrTriggerModeOn)) - { - op::error("Unable to enable trigger mode (enum entry retrieval). Aborting...", __LINE__, __FUNCTION__, __FILE__); - return -1; - } - - ptrTriggerMode->SetIntValue(ptrTriggerModeOn->GetValue()); - - op::log("Trigger mode turned back on...", op::Priority::High); - - return result; - } - catch (const Spinnaker::Exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return -1; - } - catch (const std::exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return -1; - } -} - -// This function returns the camera to a normal state by turning off trigger -// mode. -int resetTrigger(Spinnaker::GenApi::INodeMap &iNodeMap) -{ - try - { - int result = 0; - // - // Turn trigger mode back off - // - // *** NOTES *** - // Once all images have been captured, turn trigger mode back off to - // restore the camera to a clean state. - // - Spinnaker::GenApi::CEnumerationPtr ptrTriggerMode = iNodeMap.GetNode("TriggerMode"); - if (!Spinnaker::GenApi::IsAvailable(ptrTriggerMode) || !Spinnaker::GenApi::IsReadable(ptrTriggerMode)) - op::error("Unable to disable trigger mode (node retrieval). Non-fatal error...", __LINE__, __FUNCTION__, __FILE__); - - Spinnaker::GenApi::CEnumEntryPtr ptrTriggerModeOff = ptrTriggerMode->GetEntryByName("Off"); - if (!Spinnaker::GenApi::IsAvailable(ptrTriggerModeOff) || !Spinnaker::GenApi::IsReadable(ptrTriggerModeOff)) - op::error("Unable to disable trigger mode (enum entry retrieval). Non-fatal error...", __LINE__, __FUNCTION__, __FILE__); - - ptrTriggerMode->SetIntValue(ptrTriggerModeOff->GetValue()); - - // op::log("Trigger mode disabled...", op::Priority::High); - - return result; - } - catch (Spinnaker::Exception &e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return -1; - } - catch (const std::exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return -1; - } -} - -// This function acquires and displays images from each device. -std::vector acquireImages(Spinnaker::CameraList &cameraList) -{ - try - { - // Security checks - if (cameraList.GetSize() != INTRINSICS.size()) - op::checkE(cameraList.GetSize(), INTRINSICS.size(), "The number of cameras must be the same as the INTRINSICS vector size.", __LINE__, __FUNCTION__, __FILE__); - - std::vector cvMats; - - // Retrieve, convert, and return an image for each camera - // In order to work with simultaneous camera streams, nested loops are - // needed. It is important that the inner loop be the one iterating - // through the cameras; otherwise, all images will be grabbed from a - // single camera before grabbing any images from another. - - // Get cameras - std::vector cameraPtrs(cameraList.GetSize()); - for (auto i = 0u; i < cameraPtrs.size(); i++) - cameraPtrs.at(i) = cameraList.GetByIndex(i); - - std::vector imagePtrs(cameraPtrs.size()); - - // Getting frames - // Retrieve next received image and ensure image completion - // Spinnaker::ImagePtr imagePtr = cameraPtrs.at(i)->GetNextImage(); - // Clean buffer + retrieve next received image + ensure image completion - auto durationMs = 0.; - // for (auto counter = 0 ; counter < 10 ; counter++) - while (durationMs < 1.) - { - const auto begin = std::chrono::high_resolution_clock::now(); - for (auto i = 0u; i < cameraPtrs.size(); i++) - imagePtrs.at(i) = cameraPtrs.at(i)->GetNextImage(); - durationMs = std::chrono::duration_cast(std::chrono::high_resolution_clock::now()-begin).count() * 1e-6; - // op::log("Time extraction (ms): " + std::to_string(durationMs), op::Priority::High); - } - - // Original format -> RGB8 - bool imagesExtracted = true; - for (auto& imagePtr : imagePtrs) - { - if (imagePtr->IsIncomplete()) - { - op::log("Image incomplete with image status " + std::to_string(imagePtr->GetImageStatus()) + "...", - op::Priority::High, __LINE__, __FUNCTION__, __FILE__); - imagesExtracted = false; - break; - } - else - { - // Print image information - // Convert image to RGB - // Interpolation methods - // http://softwareservices.ptgrey.com/Spinnaker/latest/group___spinnaker_defs.html - // DEFAULT Default method. - // NO_COLOR_PROCESSING No color processing. - // NEAREST_NEIGHBOR Fastest but lowest quality. Equivalent to FLYCAPTURE_NEAREST_NEIGHBOR_FAST in FlyCapture. - // EDGE_SENSING Weights surrounding pixels based on localized edge orientation. - // HQ_LINEAR Well-balanced speed and quality. - // RIGOROUS Slowest but produces good results. - // IPP Multi-threaded with similar results to edge sensing. - // DIRECTIONAL_FILTER Best quality but much faster than rigorous. - // Colors - // http://softwareservices.ptgrey.com/Spinnaker/latest/group___camera_defs__h.html#ggabd5af55aaa20bcb0644c46241c2cbad1a33a1c8a1f6dbcb4a4eaaaf6d4d7ff1d1 - // PixelFormat_BGR8 - // Time tests - // const auto reps = 1e3; - // // const auto reps = 1e2; // for RIGOROUS & DIRECTIONAL_FILTER - // const auto begin = std::chrono::high_resolution_clock::now(); - // for (auto asdf = 0 ; asdf < reps ; asdf++){ - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DEFAULT); // ~ 1.5 ms but pixeled - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::NO_COLOR_PROCESSING); // ~0.5 ms but BW - imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::HQ_LINEAR); // ~6 ms, looks as good as best - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::EDGE_SENSING); // ~2 ms default << edge << best - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::RIGOROUS); // ~115, too slow - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::IPP); // ~2 ms, slightly worse than HQ_LINEAR - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DIRECTIONAL_FILTER); // ~30 ms, ideally best quality? - // imagePtr = imagePtr; - // } - // durationMs = std::chrono::duration_cast(std::chrono::high_resolution_clock::now()-begin).count() * 1e-6; - // op::log("Time conversion (ms): " + std::to_string(durationMs / reps), op::Priority::High); - } - } - - // Convert to cv::Mat - if (imagesExtracted) - { - for (auto i = 0u; i < imagePtrs.size(); i++) - { - // Baseline - // cvMats.emplace_back(pointGreyToCvMat(imagePtrs.at(i)).clone()); - // Undistort - // http://docs.opencv.org/2.4/modules/imgproc/doc/geometric_transformations.html#undistort - auto auxCvMat = pointGreyToCvMat(imagePtrs.at(i)); - cvMats.emplace_back(); - cv::undistort(auxCvMat, cvMats[i], INTRINSICS[i], DISTORTIONS[i]); - } - } - - return cvMats; - } - catch (Spinnaker::Exception &e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return {}; - } - catch (const std::exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return {}; - } -} - -// This function prints the device information of the camera from the transport -// layer; please see NodeMapInfo example for more in-depth comments on printing -// device information from the nodemap. -int printDeviceInfo(Spinnaker::GenApi::INodeMap &iNodeMap, const unsigned int camNum) -{ - int result = 0; - - op::log("Printing device information for camera " + std::to_string(camNum) + "...\n", op::Priority::High); - - Spinnaker::GenApi::FeatureList_t features; - Spinnaker::GenApi::CCategoryPtr cCategoryPtr = iNodeMap.GetNode("DeviceInformation"); - if (Spinnaker::GenApi::IsAvailable(cCategoryPtr) && Spinnaker::GenApi::IsReadable(cCategoryPtr)) - { - cCategoryPtr->GetFeatures(features); - - Spinnaker::GenApi::FeatureList_t::const_iterator it; - for (it = features.begin(); it != features.end(); ++it) - { - Spinnaker::GenApi::CNodePtr pfeatureNode = *it; - const auto cValuePtr = (Spinnaker::GenApi::CValuePtr)pfeatureNode; - op::log(pfeatureNode->GetName() + " : " + (IsReadable(cValuePtr) ? cValuePtr->ToString() : "Node not readable"), op::Priority::High); - } - } - else - op::log("Device control information not available.", op::Priority::High); - op::log(" ", op::Priority::High); - - return result; -} - -WPointGrey::WPointGrey() : - initialized{false} -{ - try - { - // Security checks - if (INTRINSICS.size() != DISTORTIONS.size()) - op::checkE(INTRINSICS.size(), DISTORTIONS.size(), "The INTRINSICS and DISTORTIONS vector should have the same size.", __LINE__, __FUNCTION__, __FILE__); - if (INTRINSICS.size() != M_EACH_CAMERA.size()) - op::checkE(INTRINSICS.size(), M_EACH_CAMERA.size(), "The INTRINSICS and M_EACH_CAMERA vector should have the same size.", __LINE__, __FUNCTION__, __FILE__); - } - catch (const std::exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } -} - -WPointGrey::~WPointGrey() -{ - try - { - if (initialized) - { - // End acquisition for each camera - // Notice that what is usually a one-step process is now two steps - // because of the additional step of selecting the camera. It is worth - // repeating that camera selection needs to be done once per loop. - // It is possible to interact with cameras through the camera list with - // GetByIndex(); this is an alternative to retrieving cameras as - // Spinnaker::CameraPtr objects that can be quick and easy for small tasks. - // - for (auto i = 0; i < mCameraList.GetSize(); i++) - mCameraList.GetByIndex(i)->EndAcquisition(); - - for (auto i = 0; i < mCameraList.GetSize(); i++) - { - // Select camera - auto cameraPtr = mCameraList.GetByIndex(i); - - // Retrieve GenICam nodemap - auto& iNodeMap = cameraPtr->GetNodeMap(); - - // // Disable chunk data - // result = disableChunkData(iNodeMap); - // // if (result < 0) - // // return result; - - // Reset trigger - auto result = resetTrigger(iNodeMap); - if (result < 0) - op::error("Error happened..." + std::to_string(result), __LINE__, __FUNCTION__, __FILE__); - - // Deinitialize each camera - // Each camera must be deinitialized separately by first - // selecting the camera and then deinitializing it. - cameraPtr->DeInit(); - } - - op::log("Completed. Releasing...", op::Priority::High); - - // Clear camera list before releasing mSystemPtr - mCameraList.Clear(); - - // Release mSystemPtr - mSystemPtr->ReleaseInstance(); - } - - op::log("Done! Exitting...", op::Priority::High); - } - catch (const Spinnaker::Exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - catch (const std::exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } -} - -void WPointGrey::initializationOnThread() -{ - try - { - initialized = true; - - // Print application build information - op::log(std::string{ "Application build date: " } + __DATE__ + " " + __TIME__, op::Priority::High); - - // Retrieve singleton reference to mSystemPtr object - mSystemPtr = Spinnaker::System::GetInstance(); - - // Retrieve list of cameras from the mSystemPtr - mCameraList = mSystemPtr->GetCameras(); - - unsigned int numCameras = mCameraList.GetSize(); - - op::log("Number of cameras detected: " + std::to_string(numCameras), op::Priority::High); - - // Finish if there are no cameras - if (numCameras == 0) - { - // Clear camera list before releasing mSystemPtr - mCameraList.Clear(); - - // Release mSystemPtr - mSystemPtr->ReleaseInstance(); - - op::log("Not enough cameras!\nPress Enter to exit...", op::Priority::High); - getchar(); - - op::error("No cameras detected.", __LINE__, __FUNCTION__, __FILE__); - } - op::log("Camera system initialized...", op::Priority::High); - - // - // Retrieve transport layer nodemaps and print device information for - // each camera - // - // *** NOTES *** - // This example retrieves information from the transport layer nodemap - // twice: once to print device information and once to grab the device - // serial number. Rather than caching the nodemap, each nodemap is - // retrieved both times as needed. - // - op::log("\n*** DEVICE INFORMATION ***\n", op::Priority::High); - - for (int i = 0; i < mCameraList.GetSize(); i++) - { - // Select camera - auto cameraPtr = mCameraList.GetByIndex(i); - - // Retrieve TL device nodemap - auto& iNodeMapTLDevice = cameraPtr->GetTLDeviceNodeMap(); - - // Print device information - auto result = printDeviceInfo(iNodeMapTLDevice, i); - if (result < 0) - op::error("Result > 0, error " + std::to_string(result) + " occurred...", __LINE__, __FUNCTION__, __FILE__); - } - - for (auto i = 0; i < mCameraList.GetSize(); i++) - { - // Select camera - auto cameraPtr = mCameraList.GetByIndex(i); - - // Initialize each camera - // You may notice that the steps in this function have more loops with - // less steps per loop; this contrasts the acquireImages() function - // which has less loops but more steps per loop. This is done for - // demonstrative purposes as both work equally well. - // Later: Each camera needs to be deinitialized once all images have been - // acquired. - cameraPtr->Init(); - - // Retrieve GenICam nodemap - // auto& iNodeMap = cameraPtr->GetNodeMap(); - - // // Configure trigger - // result = configureTrigger(iNodeMap); - // if (result < 0) - // op::error("Result > 0, error " + std::to_string(result) + " occurred...", __LINE__, __FUNCTION__, __FILE__); - - // // Configure chunk data - // result = configureChunkData(iNodeMap); - // if (result < 0) - // return result; - - // Remove buffer --> Always get newest frame - Spinnaker::GenApi::INodeMap& snodeMap = cameraPtr->GetTLStreamNodeMap(); - Spinnaker::GenApi::CEnumerationPtr ptrBufferHandlingMode = snodeMap.GetNode("StreamBufferHandlingMode"); - if (!Spinnaker::GenApi::IsAvailable(ptrBufferHandlingMode) || !Spinnaker::GenApi::IsWritable(ptrBufferHandlingMode)) - op::error("Unable to change buffer handling mode", __LINE__, __FUNCTION__, __FILE__); - - Spinnaker::GenApi::CEnumEntryPtr ptrBufferHandlingModeNewest = ptrBufferHandlingMode->GetEntryByName("NewestFirstOverwrite"); - if (!Spinnaker::GenApi::IsAvailable(ptrBufferHandlingModeNewest) || !IsReadable(ptrBufferHandlingModeNewest)) - op::error("Unable to set buffer handling mode to newest (entry 'NewestFirstOverwrite' retrieval). Aborting...", __LINE__, __FUNCTION__, __FILE__); - int64_t bufferHandlingModeNewest = ptrBufferHandlingModeNewest->GetValue(); - - ptrBufferHandlingMode->SetIntValue(bufferHandlingModeNewest); - } - - // Prepare each camera to acquire images - // - // *** NOTES *** - // For pseudo-simultaneous streaming, each camera is prepared as if it - // were just one, but in a loop. Notice that cameras are selected with - // an index. We demonstrate pseduo-simultaneous streaming because true - // simultaneous streaming would require multiple process or threads, - // which is too complex for an example. - // - // Serial numbers are the only persistent objects we gather in this - // example, which is why a std::vector is created. - std::vector strSerialNumbers(mCameraList.GetSize()); - for (auto i = 0u; i < strSerialNumbers.size(); i++) - { - // Select camera - auto cameraPtr = mCameraList.GetByIndex(i); - - // Set acquisition mode to continuous - Spinnaker::GenApi::CEnumerationPtr ptrAcquisitionMode = cameraPtr->GetNodeMap().GetNode("AcquisitionMode"); - if (!Spinnaker::GenApi::IsAvailable(ptrAcquisitionMode) || !Spinnaker::GenApi::IsWritable(ptrAcquisitionMode)) - op::error("Unable to set acquisition mode to continuous (node retrieval; camera " + std::to_string(i) + "). Aborting...", __LINE__, __FUNCTION__, __FILE__); - - Spinnaker::GenApi::CEnumEntryPtr ptrAcquisitionModeContinuous = ptrAcquisitionMode->GetEntryByName("Continuous"); - if (!Spinnaker::GenApi::IsAvailable(ptrAcquisitionModeContinuous) || !Spinnaker::GenApi::IsReadable(ptrAcquisitionModeContinuous)) - op::error("Unable to set acquisition mode to continuous (entry 'continuous' retrieval " + std::to_string(i) + "). Aborting...", __LINE__, __FUNCTION__, __FILE__); - - int64_t acquisitionModeContinuous = ptrAcquisitionModeContinuous->GetValue(); - - ptrAcquisitionMode->SetIntValue(acquisitionModeContinuous); - - op::log("Camera " + std::to_string(i) + " acquisition mode set to continuous...", op::Priority::High); - - // Begin acquiring images - cameraPtr->BeginAcquisition(); - - op::log("Camera " + std::to_string(i) + " started acquiring images...", op::Priority::High); - - // Retrieve device serial number for filename - strSerialNumbers[i] = ""; - - Spinnaker::GenApi::CStringPtr ptrStringSerial = cameraPtr->GetTLDeviceNodeMap().GetNode("DeviceSerialNumber"); - - if (Spinnaker::GenApi::IsAvailable(ptrStringSerial) && Spinnaker::GenApi::IsReadable(ptrStringSerial)) - { - strSerialNumbers[i] = ptrStringSerial->GetValue(); - op::log("Camera " + std::to_string(i) + " serial number set to " + strSerialNumbers[i].c_str() + "...", op::Priority::High); - } - op::log(" ", op::Priority::High); - } - - op::log("\nRunning for all cameras...\n\n*** IMAGE ACQUISITION ***\n", op::Priority::High); - } - catch (const Spinnaker::Exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - catch (const std::exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } -} - -std::shared_ptr> WPointGrey::workProducer() -{ - try - { - // Profiling speed - const auto profilerKey = op::Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); - // Get image from each camera - const auto cvMats = acquireImages(mCameraList); - // Images to userDatum - auto datums3d = std::make_shared>(cvMats.size()); - for (auto i = 0u ; i < cvMats.size() ; i++) - datums3d->at(i).cvInputData = cvMats.at(i); - // Profiling speed - if (!cvMats.empty()) - { - op::Profiler::timerEnd(profilerKey); - op::Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 100); - } - // Return Datum - return datums3d; - } - catch (const Spinnaker::Exception& e) - { - this->stop(); - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return nullptr; - } - catch (const std::exception& e) - { - this->stop(); - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return nullptr; - } -} diff --git a/examples_beta/openpose3d/src/reconstruction3D.cpp b/examples_beta/openpose3d/src/reconstruction3D.cpp deleted file mode 100644 index 4790f6c4..00000000 --- a/examples_beta/openpose3d/src/reconstruction3D.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include -#include -#include -#include - -double calcReprojectionError(const cv::Mat& X, const std::vector& M, const std::vector& pt2D) -{ - auto averageError = 0.; - for(unsigned int i = 0 ; i < M.size() ; i++) - { - cv::Mat imageX = M[i] * X; - imageX /= imageX.at(2,0); - const auto error = std::sqrt(std::pow(imageX.at(0,0) - pt2D[i].x,2) + std::pow(imageX.at(1,0) - pt2D[i].y,2)); - //log("Error: " + std::to_string(error)); - averageError += error; - } - return averageError / M.size(); -} - -void triangulate(cv::Mat& X, const std::vector& matrixEachCamera, const std::vector& pointOnEachCamera) -{ - // Security checks - if (matrixEachCamera.empty() || matrixEachCamera.size() != pointOnEachCamera.size()) - op::error("numberCameras.empty() || numberCameras.size() != pointOnEachCamera.size()", __LINE__, __FUNCTION__, __FILE__); - // Create and fill A - const auto numberCameras = (int)matrixEachCamera.size(); - cv::Mat A = cv::Mat::zeros(numberCameras*2, 4, CV_64F); - for (auto i = 0 ; i < numberCameras ; i++) - { - cv::Mat temp = pointOnEachCamera[i].x*matrixEachCamera[i].rowRange(2,3) - matrixEachCamera[i].rowRange(0,1); - temp.copyTo(A.rowRange(i*2,i*2+1)); - temp = pointOnEachCamera[i].y*matrixEachCamera[i].rowRange(2,3) - matrixEachCamera[i].rowRange(1,2); - temp.copyTo(A.rowRange(i*2+1,i*2+2)); - } - // SVD on A - cv::SVD svd{A}; - svd.solveZ(A,X); - X /= X.at(3); -} - -// TODO: ask Hanbyul for the missing function: TriangulationOptimization -double triangulateWithOptimization(cv::Mat& X, const std::vector& matrixEachCamera, const std::vector& pointOnEachCamera) -{ - triangulate(X, matrixEachCamera, pointOnEachCamera); - - // //if (matrixEachCamera.size() >= 3) - // //double beforeError = calcReprojectionError(&matrixEachCamera, pointOnEachCamera, X); - // double change = TriangulationOptimization(&matrixEachCamera, pointOnEachCamera, X); - // //double afterError = calcReprojectionError(&matrixEachCamera,pointOnEachCamera,X); - // //printfLog("!!Mine %.8f , inFunc %.8f \n",beforeError-afterError,change); - // return change; - return 0.; -} - -void reconstructArray(op::Array& keypoints3D, const std::vector>& keypointsVector) -{ - // Get number body parts - auto detectionMissed = false; - for (auto& keypoints : keypointsVector) - { - if (keypoints.empty()) - { - detectionMissed = true; - break; - } - } - // If at least one keypoints element not empty - if (!detectionMissed) - { - const auto numberBodyParts = keypointsVector.at(0).getSize(1); - // Create x-y vector from high score results - const auto threshold = 0.2f; - std::vector indexesUsed; - std::vector> xyPoints; - for (auto part = 0; part < numberBodyParts; part++) - { - // Create vector of points - auto missedPoint = false; - std::vector xyPointsElement; - for (auto& keypoints : keypointsVector) - { - if (keypoints[{0, part, 2}] > threshold) - xyPointsElement.emplace_back(cv::Point2d{ keypoints[{0, part, 0}], keypoints[{0, part, 1}]}); - else - { - missedPoint = true; - break; - } - } - if (!missedPoint) - { - indexesUsed.emplace_back(part); - xyPoints.emplace_back(xyPointsElement); - } - } - // 3D reconstruction - if (!xyPoints.empty()) - { - // Do 3D reconstruction - std::vector xyzPoints(xyPoints.size()); - for (auto i = 0u; i < xyPoints.size(); i++) - { - cv::Mat X; - triangulateWithOptimization(X, M_EACH_CAMERA, xyPoints[i]); - xyzPoints[i] = cv::Point3d{ X.at(0), X.at(1), X.at(2) }; - } - - // 3D points to pose - // OpenCV alternative: - // // http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#triangulatepoints - // cv::Mat reconstructedPoints{4, firstcv::Points.size(), CV_64F}; - // cv::triangulatecv::Points(cv::Mat::eye(3,4, CV_64F), M_3_1, firstcv::Points, secondcv::Points, reconstructedcv::Points); - keypoints3D = op::Array{ { 1, numberBodyParts, 4 }, 0 }; - for (auto index = 0u; index < indexesUsed.size(); index++) - { - auto& xValue = keypoints3D[{0, indexesUsed[index], 0}]; - auto& yValue = keypoints3D[{0, indexesUsed[index], 1}]; - auto& zValue = keypoints3D[{0, indexesUsed[index], 2}]; - auto& scoreValue = keypoints3D[{0, indexesUsed[index], 3}]; - if (std::isfinite(xyzPoints[index].x) && std::isfinite(xyzPoints[index].y) && std::isfinite(xyzPoints[index].z)) - { - xValue = xyzPoints[index].x; - yValue = xyzPoints[index].y; - zValue = xyzPoints[index].z; - scoreValue = 1.f; - } - } - } - } -} - -void WReconstruction3D::work(std::shared_ptr>& datumsPtr) -{ - // User's post-processing (after OpenPose processing & before OpenPose outputs) here - // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.poseKeypoints: Array with the estimated pose - try - { - // Profiling speed - const auto profilerKey = op::Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); - if (datumsPtr != nullptr && /*!datumsPtr->empty() &&*/ datumsPtr->size() == 3) - { - std::vector> poseKeypointVector; - std::vector> faceKeypointVector; - std::vector> leftHandKeypointVector; - std::vector> rightHandKeypointVector; - for (auto& datumsElement : *datumsPtr) - { - poseKeypointVector.emplace_back(datumsElement.poseKeypoints); - faceKeypointVector.emplace_back(datumsElement.faceKeypoints); - leftHandKeypointVector.emplace_back(datumsElement.handKeypoints[0]); - rightHandKeypointVector.emplace_back(datumsElement.handKeypoints[1]); - } - // Pose 3-D reconstruction - reconstructArray(datumsPtr->at(0).poseKeypoints3D, poseKeypointVector); - // Face 3-D reconstruction - reconstructArray(datumsPtr->at(0).faceKeypoints3D, faceKeypointVector); - // Left hand 3-D reconstruction - reconstructArray(datumsPtr->at(0).leftHandKeypoints3D, leftHandKeypointVector); - // Right hand 3-D reconstruction - reconstructArray(datumsPtr->at(0).rightHandKeypoints3D, rightHandKeypointVector); - // Profiling speed - op::Profiler::timerEnd(profilerKey); - op::Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 100); - } - } - catch (const std::exception& e) - { - op::log("Some kind of unexpected error happened."); - this->stop(); - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } -} diff --git a/examples_beta/openpose3d/src/renderer.cpp b/examples_beta/openpose3d/src/renderer.cpp deleted file mode 100644 index 5bbc200d..00000000 --- a/examples_beta/openpose3d/src/renderer.cpp +++ /dev/null @@ -1,416 +0,0 @@ -#include -#include -#include -#include - -struct Keypoints3D -{ - op::Array mPoseKeypoints; - op::Array mFaceKeypoints; - op::Array mLeftHandKeypoints; - op::Array mRightHandKeypoints; - bool validKeypoints; - std::mutex mutex; -}; - -enum class CameraMode { - CAM_DEFAULT, - CAM_ROTATE, - CAM_PAN, - CAM_PAN_Z -}; - -Keypoints3D gKeypoints3D; -op::PoseModel sPoseModel = op::PoseModel::COCO_18; - -CameraMode gCameraMode = CameraMode::CAM_DEFAULT; - -const std::vector LIGHT_DIFFUSE{ 1.f, 1.f, 1.f, 1.f }; // Diffuse light -const std::vector LIGHT_POSITION{ 1.f, 1.f, 1.f, 0.f }; // Infinite light location -const std::vector COLOR_DIFFUSE{ 0.5f, 0.5f, 0.5f, 1.f }; -const std::string GUI_NAME{"OpenPose 3-D Reconstruction"}; - -const auto RAD_TO_DEG = 0.0174532925199433; - -//View Change by Mouse -bool gBButton1Down = false; -auto gXClick = 0.f; -auto gYClick = 0.f; -auto gGViewDistance = -250.f; // -82.3994f; //-45; -auto gMouseXRotate = -915.f; // -63.2f; //0; -auto gMouseYRotate = -5.f; // 7.f; //60; -auto gMouseXPan = -70.f; // 0; -auto gMouseYPan = -30.f; // 0; -auto gMouseZPan = 0.f; -auto gScaleForMouseMotion = 0.1f; - -void drawConeByTwoPts(const cv::Point3f& pt1, const cv::Point3f& pt2, const float ptSize) -{ - const GLdouble x1 = pt1.x; - const GLdouble y1 = pt1.y; - const GLdouble z1 = pt1.z; - const GLdouble x2 = pt2.x; - const GLdouble y2 = pt2.y; - const GLdouble z2 = pt2.z; - - const double x = x2 - x1; - const double y = y2 - y1; - const double z = z2 - z1; - - glPushMatrix(); - - glTranslated(x1, y1, z1); - - if ((x != 0.) || (y != 0.)) - { - glRotated(std::atan2(y, x) / RAD_TO_DEG, 0., 0., 1.); - glRotated(std::atan2(std::sqrt(x*x + y*y), z) / RAD_TO_DEG, 0., 1., 0.); - } - else if (z<0) - glRotated(180, 1., 0., 0.); - - const auto height = std::sqrt((pt1.x - pt2.x)*(pt1.x - pt2.x) + (pt1.y - pt2.y)*(pt1.y - pt2.y) + (pt1.z - pt2.z)*(pt1.z - pt2.z)); - glutSolidCone(ptSize, height, 5, 5); - - glPopMatrix(); -} - -void renderHumanBody(const op::Array& keypoints, const std::vector& pairs, const std::vector colors, const float ratio) -{ - const auto person = 0; - const auto numberPeople = keypoints.getSize(0); - const auto numberBodyParts = keypoints.getSize(1); - const auto numberColors = colors.size(); - const auto xOffset = -3000; // 640.f; - const auto yOffset = 360.f; - const auto zOffset = 1000; // 360.f; - const auto xScale = 43.f; - const auto yScale = 24.f; - const auto zScale = 24.f; - - if (numberPeople > person) - //for(int person=0;person keypointColor{ - colors[colorIndex % numberColors] / 255.f, - colors[(colorIndex + 1) % numberColors] / 255.f, - colors[(colorIndex + 2) % numberColors] / 255.f, - 1.f - }; - glMaterialfv(GL_FRONT_AND_BACK, GL_DIFFUSE, COLOR_DIFFUSE.data()); - glMaterialfv(GL_FRONT_AND_BACK, GL_AMBIENT, keypointColor.data()); - // Draw circle - const auto baseIndex = 4 * part + person*numberBodyParts; - if (keypoints[baseIndex + 3] > 0) - { - cv::Point3f keypoint{ - -(keypoints[baseIndex] - xOffset) / xScale, - -(keypoints[baseIndex + 1] - yOffset) / yScale, - (keypoints[baseIndex + 2] - zOffset) / zScale - }; - // Create and add new sphere - glPushMatrix(); - glTranslatef(keypoint.x, keypoint.y, keypoint.z); - // Draw sphere - glutSolidSphere(0.5 * ratio, 20, 20); - glPopMatrix(); - } - } - - // Lines connecting each keypoint pair - for (auto pair = 0; pair < pairs.size(); pair += 2) - { - // Set color - const auto colorIndex = pairs[pair+1] * 3; - const std::vector keypointColor{ - colors[colorIndex % numberColors] / 255.f, - colors[(colorIndex + 1) % numberColors] / 255.f, - colors[(colorIndex + 2) % numberColors] / 255.f, - 1.f - }; - glMaterialfv(GL_FRONT_AND_BACK, GL_DIFFUSE, COLOR_DIFFUSE.data()); - glMaterialfv(GL_FRONT_AND_BACK, GL_AMBIENT, keypointColor.data()); - // Draw line - const auto baseIndexPairA = 4 * pairs[pair] + person*numberBodyParts; - const auto baseIndexPairB = 4 * pairs[pair + 1] + person*numberBodyParts; - if (keypoints[baseIndexPairA + 3] > 0 && keypoints[baseIndexPairB + 3] > 0) - { - cv::Point3f pairKeypointA{ - -(keypoints[baseIndexPairA] - xOffset) / xScale, - -(keypoints[baseIndexPairA + 1] - yOffset) / yScale, - (keypoints[baseIndexPairA + 2] - zOffset) / zScale - }; - cv::Point3f pairKeypointB{ - -(keypoints[baseIndexPairB] - xOffset) / xScale, - -(keypoints[baseIndexPairB + 1] - yOffset) / yScale, - (keypoints[baseIndexPairB + 2] - zOffset) / zScale - }; - drawConeByTwoPts(pairKeypointA, pairKeypointB, 0.5f * ratio); - } - } - } -} - -void initGraphics(void) -{ - // Enable a single OpenGL light - glLightfv(GL_LIGHT0, GL_AMBIENT, LIGHT_DIFFUSE.data()); - glLightfv(GL_LIGHT0, GL_DIFFUSE, LIGHT_DIFFUSE.data()); - glLightfv(GL_LIGHT0, GL_POSITION, LIGHT_POSITION.data()); - glEnable(GL_LIGHT0); - glEnable(GL_LIGHTING); - - // Use depth buffering for hidden surface elimination - glEnable(GL_DEPTH_TEST); - - // Setup the view of the cube - glMatrixMode(GL_PROJECTION); - gluPerspective( /* field of view in degree */ 40.0, - /* aspect ratio */ 1.0, - /* Z near */ 1.0, /* Z far */ 1000.0); - glMatrixMode(GL_MODELVIEW); - gluLookAt( - 0.0, 0.0, 5.0, // eye is at (0,0,5) - 0.0, 0.0, 0.0, // center is at (0,0,0) - 0.0, 1.0, 0. // up is in positive Y direction - ); - - // Adjust cube position to be asthetic angle - glTranslatef(0.0, 0.0, -1.0); - glRotatef(60, 1.0, 0.0, 0.0); - glRotatef(-20, 0.0, 0.0, 1.0); - - glColorMaterial(GL_FRONT, GL_DIFFUSE); - glEnable(GL_COLOR_MATERIAL); -} - -// this is the actual idle function -void idleFunc() -{ - glutPostRedisplay(); - glutSwapBuffers(); -} - -void renderFloor() -{ - glDisable(GL_LIGHTING); - - const cv::Point3f gGloorCenter{ 0,0,0 }; //ankle - const cv::Point3f Noise{ 0,1,0 }; - - cv::Point3f upright = Noise - gGloorCenter; - upright = 1.0 / sqrt(upright.x *upright.x + upright.y *upright.y + upright.z *upright.z)*upright; - const cv::Point3f gGloorAxis2 = cv::Point3f{ 1,0,0 }.cross(upright); - const cv::Point3f gGloorAxis1 = gGloorAxis2.cross(upright); - - const auto gridNum = 10; - const auto width = 50.;//sqrt(Distance(gGloorPts.front(),gGloorCenter)*2 /gridNum) * 1.2; - const cv::Point3f origin = gGloorCenter - gGloorAxis1*(width*gridNum / 2) - gGloorAxis2*(width*gridNum / 2); - const cv::Point3f axis1 = gGloorAxis1 * width; - const cv::Point3f axis2 = gGloorAxis2 * width; - for (auto y = 0; y <= gridNum; ++y) - { - for (auto x = 0; x <= gridNum; ++x) - { - if ((x + y) % 2 == 0) - glColor4f(0.2f, 0.2f, 0.2f, 1.f); //black - else - glColor4f(0.5f, 0.5f, 0.5f, 1.f); //grey - - const cv::Point3f p1 = origin + axis1*x + axis2*y; - const cv::Point3f p2 = p1 + axis1; - const cv::Point3f p3 = p1 + axis2; - const cv::Point3f p4 = p1 + axis1 + axis2; - - glBegin(GL_QUADS); - - glVertex3f(p1.x, p1.y, p1.z); - glVertex3f(p2.x, p2.y, p2.z); - glVertex3f(p4.x, p4.y, p4.z); - glVertex3f(p3.x, p3.y, p3.z); - glEnd(); - } - } - glEnable(GL_LIGHTING); -} - -void renderMain(void) -{ - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - - glLoadIdentity(); - //gluLookAt(0,0,0, 0, 0, 1, 0, -1, 0); - gluLookAt( - 0.0, 0.0, 5.0, // eye is at (0,0,5) - 0.0, 0.0, 0.0, // center is at (0,0,0) - 0.0, 1.0, 0. // up is in positive Y direction - ); - - glTranslatef(0, 0, gGViewDistance); - glRotatef(-gMouseYRotate, 1.f, 0.f, 0.f); - glRotatef(-gMouseXRotate, 0.f, 1.f, 0.f); - - glTranslatef(-gMouseXPan, gMouseYPan, -gMouseZPan); - - renderFloor(); - std::unique_lock lock{gKeypoints3D.mutex}; - if (gKeypoints3D.validKeypoints) - { - renderHumanBody(gKeypoints3D.mPoseKeypoints, op::getPoseBodyPartPairsRender(sPoseModel), op::getPoseColors(sPoseModel), 1.f); - renderHumanBody(gKeypoints3D.mFaceKeypoints, op::FACE_PAIRS_RENDER, op::FACE_COLORS_RENDER, 0.5f); - renderHumanBody(gKeypoints3D.mLeftHandKeypoints, op::HAND_PAIRS_RENDER, op::HAND_COLORS_RENDER, 0.5f); - renderHumanBody(gKeypoints3D.mRightHandKeypoints, op::HAND_PAIRS_RENDER, op::HAND_COLORS_RENDER, 0.5f); - } - lock.unlock(); - - glutSwapBuffers(); -} - -void mouseButton(const int button, const int state, const int x, const int y) -{ - - if (button == 3 || button == 4) //mouse wheel - { - if (button == 3) //zoom in - gGViewDistance += 10 * gScaleForMouseMotion; - else //zoom out - gGViewDistance -= 10 * gScaleForMouseMotion; - op::log("gGViewDistance: " + std::to_string(gGViewDistance)); - } - else - { - if (button == GLUT_LEFT_BUTTON) - { - gBButton1Down = (state == GLUT_DOWN) ? 1 : 0; - gXClick = (float)x; - gYClick = (float)y; - - if (glutGetModifiers() == GLUT_ACTIVE_SHIFT) - gCameraMode = CameraMode::CAM_PAN; - else - gCameraMode = CameraMode::CAM_ROTATE; - } - op::log("Clicked: [" + std::to_string(gXClick) + "," + std::to_string(gYClick) + "]"); - } - glutPostRedisplay(); -} - -void mouseMotion(const int x, const int y) -{ - - // If button1 pressed, zoom in/out if mouse is moved up/down. - if (gBButton1Down) - { - if (gCameraMode == CameraMode::CAM_ROTATE) - { - gMouseXRotate += (x - gXClick)*0.2f; - gMouseYRotate -= (y - gYClick)*0.2f; - } - else if (gCameraMode == CameraMode::CAM_PAN) - { - gMouseXPan -= (x - gXClick) / 2 * gScaleForMouseMotion; - gMouseYPan -= (y - gYClick) / 2 * gScaleForMouseMotion; - } - else if (gCameraMode == CameraMode::CAM_PAN_Z) - { - auto dist = std::sqrt(pow((x - gXClick), 2.0f) + pow((y - gYClick), 2.0f)); - if (y < gYClick) - dist *= -1; - gMouseZPan -= dist / 5 * gScaleForMouseMotion; - } - - gXClick = (float)x; - gYClick = (float)y; - - glutPostRedisplay(); - op::log("gMouseXRotate = " + std::to_string(gMouseXRotate)); - op::log("gMouseYRotate = " + std::to_string(gMouseYRotate)); - op::log("gMouseXPan = " + std::to_string(gMouseXPan)); - op::log("gMouseYPan = " + std::to_string(gMouseYPan)); - op::log("gMouseZPan = " + std::to_string(gMouseZPan)); - } -} - -WRender3D::WRender3D(const op::PoseModel poseModel) -{ - // Update sPoseModel - sPoseModel = poseModel; - // Init display - cv::imshow(GUI_NAME, cv::Mat( 500, 500, CV_8UC3, cv::Scalar{ 0,0,0 } )); - //Run OpenGL - mRenderThread = std::thread{ &WRender3D::visualizationThread, this }; -} - -void WRender3D::workConsumer(const std::shared_ptr>& datumsPtr) -{ - try - { - // Profiling speed - const auto profilerKey = op::Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); - // User's displaying/saving/other processing here - // datum.cvOutputData: rendered frame with pose or heatmaps - // datum.poseKeypoints: Array with the estimated pose - if (datumsPtr != nullptr && !datumsPtr->empty()) - { - cv::Mat cvMat = datumsPtr->at(0).cvOutputData.clone(); - for (auto i = 1u; i < datumsPtr->size(); i++) - cv::hconcat(cvMat, datumsPtr->at(i).cvOutputData, cvMat); - // while (cvMat.cols > 1500 || cvMat.rows > 1500) - while (cvMat.cols > 1920 || cvMat.rows > 1920) - // while (cvMat.rows > 3500) - cv::pyrDown(cvMat, cvMat); - // Display all views - cv::imshow(GUI_NAME, cvMat); - cv::resizeWindow(GUI_NAME, cvMat.cols, cvMat.rows); - // OpenGL Rendering - std::unique_lock lock{gKeypoints3D.mutex}; - gKeypoints3D.mPoseKeypoints = datumsPtr->at(0).poseKeypoints3D; - gKeypoints3D.mFaceKeypoints = datumsPtr->at(0).faceKeypoints3D; - gKeypoints3D.mLeftHandKeypoints = datumsPtr->at(0).leftHandKeypoints3D; - gKeypoints3D.mRightHandKeypoints = datumsPtr->at(0).rightHandKeypoints3D; - gKeypoints3D.validKeypoints = true; - lock.unlock(); - // Profiling speed - op::Profiler::timerEnd(profilerKey); - op::Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, 100); - } - // Render images - cv::waitKey(1); // It sleeps 1 ms just to let the user see the output. Change to 33ms for normal 30 fps display if too fast - } - catch (const std::exception& e) - { - op::log("Some kind of unexpected error happened."); - this->stop(); - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } -} - -void WRender3D::visualizationThread() -{ - char *my_argv[] = { NULL }; - int my_argc = 0; - glutInit(&my_argc, my_argv); - - // setup the size, position, and display mode for new windows - glutInitWindowSize(1280, 720); - glutInitWindowPosition(200, 0); - // glutSetOption(GLUT_MULTISAMPLE,8); - glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH | GLUT_MULTISAMPLE); - - // create and set up a window - glutCreateWindow(GUI_NAME.c_str()); - initGraphics(); - glutDisplayFunc(renderMain); - glutMouseFunc(mouseButton); - glutMotionFunc(mouseMotion); - glutIdleFunc(idleFunc); - - glutMainLoop(); - - this->stop(); -} diff --git a/include/openpose/experimental/3d/cameraParameters.hpp b/include/openpose/experimental/3d/cameraParameters.hpp new file mode 100644 index 00000000..ab1732ed --- /dev/null +++ b/include/openpose/experimental/3d/cameraParameters.hpp @@ -0,0 +1,20 @@ +#ifndef OPENPOSE_EXPERIMENTAL_3D_CAMERA_PARAMETERS_HPP +#define OPENPOSE_EXPERIMENTAL_3D_CAMERA_PARAMETERS_HPP + +#include +#include + +namespace op +{ + OP_API const cv::Mat getIntrinsics(const int cameraIndex); + + OP_API const cv::Mat getDistorsion(const int cameraIndex); + + OP_API const cv::Mat getM(const int cameraIndex); + + OP_API std::vector getMs(); + + OP_API unsigned long long getNumberCameras(); +} + +#endif // OPENPOSE_EXPERIMENTAL_3D_CAMERA_PARAMETERS_HPP diff --git a/include/openpose/experimental/3d/datum3D.hpp b/include/openpose/experimental/3d/datum3D.hpp new file mode 100644 index 00000000..a5732644 --- /dev/null +++ b/include/openpose/experimental/3d/datum3D.hpp @@ -0,0 +1,20 @@ +#ifndef OPENPOSE_EXPERIMENTAL_3D_DATUM_3D_HPP +#define OPENPOSE_EXPERIMENTAL_3D_DATUM_3D_HPP + +#include +#include + +namespace op +{ + // Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from Datum + // See `examples/tutorial_wrapper/` for more details + struct OP_API Datum3D : public Datum + { + Array poseKeypoints3D; + Array faceKeypoints3D; + Array leftHandKeypoints3D; + Array rightHandKeypoints3D; + }; +} + +#endif // OPENPOSE_EXPERIMENTAL_3D_DATUM_3D_HPP diff --git a/include/openpose/experimental/3d/headers.hpp b/include/openpose/experimental/3d/headers.hpp new file mode 100644 index 00000000..59ee5b12 --- /dev/null +++ b/include/openpose/experimental/3d/headers.hpp @@ -0,0 +1,15 @@ +#ifndef OPENPOSE_EXPERIMENTAL_3D_HEADERS_HPP +#define OPENPOSE_EXPERIMENTAL_3D_HEADERS_HPP + +// Camera intrinsic and extrinsic parameters +#include +// Datum3D +#include +// PointGrey cameras +#include +// 3D reconstruction +#include +// OpenGL Renderer +#include + +#endif // OPENPOSE_EXPERIMENTAL_3D_HEADERS_HPP diff --git a/include/openpose/experimental/3d/pointGrey.hpp b/include/openpose/experimental/3d/pointGrey.hpp new file mode 100644 index 00000000..01d94604 --- /dev/null +++ b/include/openpose/experimental/3d/pointGrey.hpp @@ -0,0 +1,39 @@ +#ifndef OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP +#define OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP + +#include +#include +#include + +namespace op +{ + // Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from WorkerProducer. + // This worker: + // 1. Set hardware trigger and the buffer to get the latest obtained frame. + // 2. Read images from FLIR cameras. + // 3. Turn them into std::vector. + // 4. Return the resulting images wrapped into a std::shared_ptr>. + // The HW trigger + reading FLIR camera code is highly based on the Spinnaker SDK examples `AcquisitionMultipleCamera` + // and specially `Trigger` + // (located in `src/`). See them for more details about the cameras. + // See `examples/tutorial_wrapper/` for more details about inhering the WorkerProducer class. + class OP_API WPointGrey : public WorkerProducer>> + { + public: + WPointGrey(); + + ~WPointGrey(); + + void initializationOnThread(); + + std::shared_ptr> workProducer(); + + private: + // PIMPL idiom + // http://www.cppsamples.com/common-tasks/pimpl.html + struct ImplWPointGrey; + std::unique_ptr upImpl; + }; +} + +#endif // OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP diff --git a/include/openpose/experimental/3d/reconstruction3D.hpp b/include/openpose/experimental/3d/reconstruction3D.hpp new file mode 100644 index 00000000..3bf35c4c --- /dev/null +++ b/include/openpose/experimental/3d/reconstruction3D.hpp @@ -0,0 +1,26 @@ +#ifndef OPENPOSE_EXPERIMENTAL_3D_RECONSTRUCTION_3D_HPP +#define OPENPOSE_EXPERIMENTAL_3D_RECONSTRUCTION_3D_HPP + +#include +#include +#include + +namespace op +{ + // Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from Worker. + // This worker will do 3-D reconstruction + // We apply the simple Direct linear transformation (DLT) algorithm, asumming each keypoint (e.g. right hip) is seen + // by all the cameras. + // No non-linear minimization used, and if some camera misses the point, it is not reconstructed. + // See `examples/tutorial_wrapper/` for more details about inhering the Worker class and using it for post-processing + // purposes. + class OP_API WReconstruction3D : public Worker>> + { + public: + void initializationOnThread() {} + + void work(std::shared_ptr>& datumsPtr); + }; +} + +#endif // OPENPOSE_EXPERIMENTAL_3D_RECONSTRUCTION_3D_HPP diff --git a/include/openpose/experimental/3d/renderer.hpp b/include/openpose/experimental/3d/renderer.hpp new file mode 100644 index 00000000..5af3773c --- /dev/null +++ b/include/openpose/experimental/3d/renderer.hpp @@ -0,0 +1,29 @@ +#ifndef OPENPOSE_EXPERIMENTAL_3D_RENDERER_HPP +#define OPENPOSE_EXPERIMENTAL_3D_RENDERER_HPP + +#include +#include +#include +#include +#include + +namespace op +{ + // This worker will do 3-D rendering + class OP_API WRender3D : public WorkerConsumer>> + { + public: + WRender3D(const PoseModel poseModel = PoseModel::COCO_18); + + void initializationOnThread() {} + + void workConsumer(const std::shared_ptr>& datumsPtr); + + private: + std::thread mRenderThread; + + void visualizationThread(); + }; +} + +#endif // OPENPOSE_EXPERIMENTAL_3D_RENDERER_HPP diff --git a/include/openpose/experimental/headers.hpp b/include/openpose/experimental/headers.hpp index b34802d5..56959aa8 100644 --- a/include/openpose/experimental/headers.hpp +++ b/include/openpose/experimental/headers.hpp @@ -2,7 +2,7 @@ #define OPENPOSE_EXPERIMENTAL_HEADERS_HPP // experimental modules -// #include +#include #include #endif // OPENPOSE_EXPERIMENTAL_HEADERS_HPP diff --git a/src/openpose/CMakeLists.txt b/src/openpose/CMakeLists.txt index 576e8331..f3488e3c 100644 --- a/src/openpose/CMakeLists.txt +++ b/src/openpose/CMakeLists.txt @@ -25,7 +25,7 @@ endif () # Ubuntu if(UNIX AND NOT APPLE) target_link_libraries(openpose ${OpenCV_LIBS} ${Caffe_LIBS} - ${GFLAGS_LIBRARY} ${GLOG_LIBRARY} ${MKL_LIBS}) + ${GFLAGS_LIBRARY} ${GLOG_LIBRARY} ${MKL_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB}) if (CMAKE_COMPILER_IS_GNUCXX) foreach (SUB_DIR ${SUB_DIRS}) set_target_properties(openpose_${SUB_DIR} PROPERTIES COMPILE_FLAGS ${OP_CXX_FLAGS}) @@ -35,7 +35,7 @@ if(UNIX AND NOT APPLE) # Windows elseif (WIN32) set_property(TARGET openpose PROPERTY DEBUG_POSTFIX d) - target_link_libraries(openpose ${OpenCV_LIBS} ${Caffe_LIBS} + target_link_libraries(openpose ${OpenCV_LIBS} ${Caffe_LIBS} ${GLUT_LIBRARY} ${SPINNAKER_LIB} debug ${GFLAGS_LIBRARY_DEBUG} optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GLOG_LIBRARY_DEBUG} optimized ${GLOG_LIBRARY_RELEASE}) if (${DL_FRAMEWORK} MATCHES "CAFFE") @@ -52,6 +52,10 @@ elseif (WIN32) source_group("Header Files\\${SUB_DIR}" FILES ${SUB_MOD_HEADERS}) endforeach (SUB_DIR ${SUB_DIRS}) set_target_properties(openpose PROPERTIES COMPILE_FLAGS -DOP_EXPORTS) + # Properties->General->Output Directory + set_property(TARGET openpose PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + set_property(TARGET openpose PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_BINARY_DIR}/$(Platform)/$(Configuration)) + endif (UNIX AND NOT APPLE) if (UNIX AND NOT APPLE) diff --git a/src/openpose/experimental/3d/CMakeLists.txt b/src/openpose/experimental/3d/CMakeLists.txt new file mode 100644 index 00000000..b510b50f --- /dev/null +++ b/src/openpose/experimental/3d/CMakeLists.txt @@ -0,0 +1,22 @@ +set(SOURCES_OP_3D + cameraParameters.cpp + pointGrey.cpp + reconstruction3D.cpp + renderer.cpp) + +include(${CMAKE_SOURCE_DIR}/cmake/Utils.cmake) +prepend(SOURCES_OP_3D_WITH_CP ${CMAKE_CURRENT_SOURCE_DIR} ${SOURCES_OP_3D}) +set(SOURCES_OP_3D_WITH_CP ${SOURCES_OP_3D_WITH_CP} PARENT_SCOPE) +set(SOURCES_OPENPOSE ${SOURCES_OPENPOSE} ${SOURCES_OP_3D_WITH_CP} PARENT_SCOPE) + +if (UNIX AND NOT APPLE) + add_library(openpose_3d ${SOURCES_OP_3D}) + + set_target_properties(openpose_3d PROPERTIES COMPILE_FLAGS ${OP_CXX_FLAGS}) + + install(TARGETS openpose_3d + EXPORT OpenPose + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib/openpose) +endif (UNIX AND NOT APPLE) diff --git a/src/openpose/experimental/3d/cameraParameters.cpp b/src/openpose/experimental/3d/cameraParameters.cpp new file mode 100644 index 00000000..0a98ca8b --- /dev/null +++ b/src/openpose/experimental/3d/cameraParameters.cpp @@ -0,0 +1,136 @@ +#include +#include +#include + +namespace op +{ + // User configurable code + // Intrinsic and distortion parameters + // Camera 1 parameters + const cv::Mat INTRINSIC_1 = (cv::Mat_(3, 3) << 817.93481631740565, 0, 600.70689997785121, + 0, 816.51774059837908, 517.84529566329593, + 0, 0, 1); + const cv::Mat DISTORTION_1 = (cv::Mat_(8, 1) << + -1.8102158829399091, 9.1966147162623262, -0.00044293900343777355, 0.0013638377686816653, 1.3303863414979364, -1.418905163635487, 8.4725535468475819, 4.7911023525901033); + // Camera 2 parameters + const cv::Mat INTRINSIC_2 = (cv::Mat_(3, 3) << 816.20921132436638, 0, 612.67087968681585, + 0, 816.18292222910486, 530.47901782670431, + 0, 0, 1); + const cv::Mat DISTORTION_2 = (cv::Mat_(8, 1) << + -5.1088507540294881, 133.63995617304997, -0.0010048069080912836, 0.00018825291386406282, 20.688286893903879, -4.7604289550474768, 132.42412342224557, 70.01195364029752); + const cv::Mat INTRINSIC_3 = (cv::Mat_(3, 3) << 798.42980806905666, 0, 646.48130011561727, + 0, 798.46535448393979, 523.91590563194586, + 0, 0, 1); + // Camera 3 + const cv::Mat DISTORTION_3 = (cv::Mat_(8, 1) << + -0.57530495294002304, -0.54721992620722582, -0.00037614702677289967, -0.00081995658363481598, -0.020321660897680775, -0.18040544059116842, -0.87724444571603022, -0.13136636671099691); + + // Extrinsic parameters - rotation and pose orientation between cameras + // Camera 1 + const cv::Mat M_1_1 = (cv::Mat_(3, 4) << 1, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0); + // Not working on Windows + // const cv::Mat M_1_1 = cv::Mat::eye(3, 4, CV_64F); + // From camera 1 to 2 + const cv::Mat M_1_2 = (cv::Mat_(3, 4) << 0.999962504862692, -0.00165862051503619, 0.00849928507093793, -238.301309354482, + 0.00176155163779584, 0.999925029704659, -0.0121174215889211, 4.75863886121558, + -0.00847854967298925, 0.0121319391740716, 0.999890459124058, 15.9219925821916); + // From camera 1 to 3 + const cv::Mat M_1_3 = (cv::Mat_(3, 4) << 0.995809442124071, -0.000473104796892308, 0.0914512501193800, -461.301274485705, + 0.00165046455210419, 0.999916727562850, -0.0127989806923977, 6.22648121362088, + -0.0914375794917412, 0.0128962828696210, 0.995727299487585, 63.4911132860733); + // From camera 2 to 3 + const cv::Mat M_2_3 = (cv::Mat_(3, 4) << 0.999644115423621, -0.00194501088674130, -0.0266056278177532, -235.236375502202, + 0.00201646110733780, 0.999994431880356, 0.00265896462686206, 9.52238656728889, + 0.0266003079592876, -0.00271166755609303, 0.999642471324391, -4.23534963077479); + + + + // Do not modify this code + const std::vector INTRINSICS{ INTRINSIC_1, INTRINSIC_2, INTRINSIC_3 }; + const std::vector DISTORTIONS{ DISTORTION_1, DISTORTION_2, DISTORTION_3 }; + const std::vector M{ M_1_1, M_1_2, M_1_3 }; + // Not working on Windows + // const std::vector M_EACH_CAMERA{ + // INTRINSIC_1 * M_1_1, + // INTRINSIC_2 * M_1_2, + // INTRINSIC_3 * M_1_3 + // }; + + const cv::Mat getIntrinsics(const int cameraIndex) + { + try + { + return INTRINSICS[cameraIndex]; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return cv::Mat{}; + } + } + + const cv::Mat getDistorsion(const int cameraIndex) + { + try + { + return DISTORTIONS[cameraIndex]; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return cv::Mat{}; + } + } + + const cv::Mat getM(const int cameraIndex) + { + try + { + return INTRINSICS[cameraIndex] * M[cameraIndex]; + // return M_EACH_CAMERA[cameraIndex]; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return cv::Mat{}; + } + } + std::vector getMs() + { + try + { + // Security checks + if (INTRINSICS.size() != DISTORTIONS.size()) + error("The INTRINSICS and DISTORTIONS vector should have the same size.", + __LINE__, __FUNCTION__, __FILE__); + if (INTRINSICS.size() != M.size()) + error("The INTRINSICS and M_EACH_CAMERA vector should have the same size.", + __LINE__, __FUNCTION__, __FILE__); + + std::vector result(INTRINSICS.size()); + for (auto i = 0u ; i < result.size(); i++) + result[i] = getM(i); + return result; + // return M_EACH_CAMERA; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return{}; + } + } + unsigned long long getNumberCameras() + { + try + { + return INTRINSICS.size(); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return 0; + } + } +} diff --git a/src/openpose/experimental/3d/pointGrey.cpp b/src/openpose/experimental/3d/pointGrey.cpp new file mode 100644 index 00000000..0c477c34 --- /dev/null +++ b/src/openpose/experimental/3d/pointGrey.cpp @@ -0,0 +1,639 @@ +#include +#include +#include +#ifdef BUILD_MODULE_3D + #include +#endif +#include +#include +#include + +namespace op +{ + #ifdef BUILD_MODULE_3D + /* + * This function converts between Spinnaker::ImagePtr container to cv::Mat container used in OpenCV. + */ + cv::Mat pointGreyToCvMat(const Spinnaker::ImagePtr &imagePtr) + { + try + { + const auto XPadding = imagePtr->GetXPadding(); + const auto YPadding = imagePtr->GetYPadding(); + const auto rowsize = imagePtr->GetWidth(); + const auto colsize = imagePtr->GetHeight(); + + // Image data contains padding. When allocating cv::Mat container size, you need to account for the X,Y + // image data padding. + return cv::Mat((int)(colsize + YPadding), (int)(rowsize + XPadding), CV_8UC3, imagePtr->GetData(), + imagePtr->GetStride()); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return cv::Mat(); + } + } + + // This function configures the camera to use a trigger. First, trigger mode is + // set to off in order to select the trigger source. Once the trigger source + // has been selected, trigger mode is then enabled, which has the camera + // capture only a single image upon the execution of the chosen trigger. + int configureTrigger(Spinnaker::GenApi::INodeMap &iNodeMap) + { + try + { + int result = 0; + log("*** CONFIGURING TRIGGER ***", Priority::High); + log("Configuring hardware trigger...", Priority::High); + // Ensure trigger mode off + // *** NOTES *** + // The trigger must be disabled in order to configure whether the source + // is software or hardware. + Spinnaker::GenApi::CEnumerationPtr ptrTriggerMode = iNodeMap.GetNode("TriggerMode"); + if (!Spinnaker::GenApi::IsAvailable(ptrTriggerMode) || !Spinnaker::GenApi::IsReadable(ptrTriggerMode)) + error("Unable to disable trigger mode (node retrieval). Aborting...", + __LINE__, __FUNCTION__, __FILE__); + + Spinnaker::GenApi::CEnumEntryPtr ptrTriggerModeOff = ptrTriggerMode->GetEntryByName("Off"); + if (!Spinnaker::GenApi::IsAvailable(ptrTriggerModeOff) || !Spinnaker::GenApi::IsReadable(ptrTriggerModeOff)) + error("Unable to disable trigger mode (enum entry retrieval). Aborting...", + __LINE__, __FUNCTION__, __FILE__); + + ptrTriggerMode->SetIntValue(ptrTriggerModeOff->GetValue()); + + log("Trigger mode disabled...", Priority::High); + + // Select trigger source + // *** NOTES *** + // The trigger source must be set to hardware or software while trigger + // mode is off. + Spinnaker::GenApi::CEnumerationPtr ptrTriggerSource = iNodeMap.GetNode("TriggerSource"); + if (!Spinnaker::GenApi::IsAvailable(ptrTriggerSource) || !Spinnaker::GenApi::IsWritable(ptrTriggerSource)) + error("Unable to set trigger mode (node retrieval). Aborting...", __LINE__, __FUNCTION__, __FILE__); + + // Set trigger mode to hardware ('Line0') + Spinnaker::GenApi::CEnumEntryPtr ptrTriggerSourceHardware = ptrTriggerSource->GetEntryByName("Line0"); + if (!Spinnaker::GenApi::IsAvailable(ptrTriggerSourceHardware) + || !Spinnaker::GenApi::IsReadable(ptrTriggerSourceHardware)) + error("Unable to set trigger mode (enum entry retrieval). Aborting...", + __LINE__, __FUNCTION__, __FILE__); + + ptrTriggerSource->SetIntValue(ptrTriggerSourceHardware->GetValue()); + + log("Trigger source set to hardware...", Priority::High); + + // Turn trigger mode on + // *** LATER *** + // Once the appropriate trigger source has been set, turn trigger mode + // on in order to retrieve images using the trigger. + Spinnaker::GenApi::CEnumEntryPtr ptrTriggerModeOn = ptrTriggerMode->GetEntryByName("On"); + if (!Spinnaker::GenApi::IsAvailable(ptrTriggerModeOn) || !Spinnaker::GenApi::IsReadable(ptrTriggerModeOn)) + { + error("Unable to enable trigger mode (enum entry retrieval). Aborting...", + __LINE__, __FUNCTION__, __FILE__); + return -1; + } + + ptrTriggerMode->SetIntValue(ptrTriggerModeOn->GetValue()); + + log("Trigger mode turned back on...", Priority::High); + + return result; + } + catch (const Spinnaker::Exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return -1; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return -1; + } + } + + // This function returns the camera to a normal state by turning off trigger + // mode. + int resetTrigger(Spinnaker::GenApi::INodeMap &iNodeMap) + { + try + { + int result = 0; + // + // Turn trigger mode back off + // + // *** NOTES *** + // Once all images have been captured, turn trigger mode back off to + // restore the camera to a clean state. + // + Spinnaker::GenApi::CEnumerationPtr ptrTriggerMode = iNodeMap.GetNode("TriggerMode"); + if (!Spinnaker::GenApi::IsAvailable(ptrTriggerMode) || !Spinnaker::GenApi::IsReadable(ptrTriggerMode)) + error("Unable to disable trigger mode (node retrieval). Non-fatal error...", + __LINE__, __FUNCTION__, __FILE__); + + Spinnaker::GenApi::CEnumEntryPtr ptrTriggerModeOff = ptrTriggerMode->GetEntryByName("Off"); + if (!Spinnaker::GenApi::IsAvailable(ptrTriggerModeOff) || !Spinnaker::GenApi::IsReadable(ptrTriggerModeOff)) + error("Unable to disable trigger mode (enum entry retrieval). Non-fatal error...", + __LINE__, __FUNCTION__, __FILE__); + + ptrTriggerMode->SetIntValue(ptrTriggerModeOff->GetValue()); + + // log("Trigger mode disabled...", Priority::High); + + return result; + } + catch (Spinnaker::Exception &e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return -1; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return -1; + } + } + + // This function acquires and displays images from each device. + std::vector acquireImages(Spinnaker::CameraList &cameraList) + { + try + { + // Security checks + if ((unsigned long long) cameraList.GetSize() != getNumberCameras()) + error("The number of cameras must be the same as the INTRINSICS vector size.", + __LINE__, __FUNCTION__, __FILE__); + + std::vector cvMats; + + // Retrieve, convert, and return an image for each camera + // In order to work with simultaneous camera streams, nested loops are + // needed. It is important that the inner loop be the one iterating + // through the cameras; otherwise, all images will be grabbed from a + // single camera before grabbing any images from another. + + // Get cameras + std::vector cameraPtrs(cameraList.GetSize()); + for (auto i = 0u; i < cameraPtrs.size(); i++) + cameraPtrs.at(i) = cameraList.GetByIndex(i); + + std::vector imagePtrs(cameraPtrs.size()); + + // Getting frames + // Retrieve next received image and ensure image completion + // Spinnaker::ImagePtr imagePtr = cameraPtrs.at(i)->GetNextImage(); + // Clean buffer + retrieve next received image + ensure image completion + auto durationMs = 0.; + // for (auto counter = 0 ; counter < 10 ; counter++) + while (durationMs < 1.) + { + const auto begin = std::chrono::high_resolution_clock::now(); + for (auto i = 0u; i < cameraPtrs.size(); i++) + imagePtrs.at(i) = cameraPtrs.at(i)->GetNextImage(); + durationMs = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now()-begin + ).count() * 1e-6; + // log("Time extraction (ms): " + std::to_string(durationMs), Priority::High); + } + + // Original format -> RGB8 + bool imagesExtracted = true; + for (auto& imagePtr : imagePtrs) + { + if (imagePtr->IsIncomplete()) + { + log("Image incomplete with image status " + std::to_string(imagePtr->GetImageStatus()) + "...", + Priority::High, __LINE__, __FUNCTION__, __FILE__); + imagesExtracted = false; + break; + } + else + { + // Print image information + // Convert image to RGB + // Interpolation methods + // http://softwareservices.ptgrey.com/Spinnaker/latest/group___spinnaker_defs.html + // DEFAULT Default method. + // NO_COLOR_PROCESSING No color processing. + // NEAREST_NEIGHBOR Fastest but lowest quality. Equivalent to FLYCAPTURE_NEAREST_NEIGHBOR_FAST + // in FlyCapture. + // EDGE_SENSING Weights surrounding pixels based on localized edge orientation. + // HQ_LINEAR Well-balanced speed and quality. + // RIGOROUS Slowest but produces good results. + // IPP Multi-threaded with similar results to edge sensing. + // DIRECTIONAL_FILTER Best quality but much faster than rigorous. + // Colors + // http://softwareservices.ptgrey.com/Spinnaker/latest/group___camera_defs__h.html#ggabd5af55aaa20bcb0644c46241c2cbad1a33a1c8a1f6dbcb4a4eaaaf6d4d7ff1d1 + // PixelFormat_BGR8 + // Time tests + // const auto reps = 1e3; + // // const auto reps = 1e2; // for RIGOROUS & DIRECTIONAL_FILTER + // const auto begin = std::chrono::high_resolution_clock::now(); + // for (auto asdf = 0 ; asdf < reps ; asdf++){ + // ~ 1.5 ms but pixeled + // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DEFAULT); + // ~0.5 ms but BW + // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::NO_COLOR_PROCESSING); + // ~6 ms, looks as good as best + imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::HQ_LINEAR); + // ~2 ms default << edge << best + // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::EDGE_SENSING); + // ~115, too slow + // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::RIGOROUS); + // ~2 ms, slightly worse than HQ_LINEAR + // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::IPP); + // ~30 ms, ideally best quality? + // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DIRECTIONAL_FILTER); + // imagePtr = imagePtr; + // } + // durationMs = std::chrono::duration_cast( + // std::chrono::high_resolution_clock::now()-begin + // ).count() * 1e-6; + // log("Time conversion (ms): " + std::to_string(durationMs / reps), Priority::High); + } + } + + // Convert to cv::Mat + if (imagesExtracted) + { + for (auto i = 0u; i < imagePtrs.size(); i++) + { + // Baseline + // cvMats.emplace_back(pointGreyToCvMat(imagePtrs.at(i)).clone()); + // Undistort + // http://docs.opencv.org/2.4/modules/imgproc/doc/geometric_transformations.html#undistort + auto auxCvMat = pointGreyToCvMat(imagePtrs.at(i)); + cvMats.emplace_back(); + cv::undistort(auxCvMat, cvMats[i], getIntrinsics(i), getDistorsion(i)); + } + } + + return cvMats; + } + catch (Spinnaker::Exception &e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return {}; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return {}; + } + } + + // This function prints the device information of the camera from the transport + // layer; please see NodeMapInfo example for more in-depth comments on printing + // device information from the nodemap. + int printDeviceInfo(Spinnaker::GenApi::INodeMap &iNodeMap, const unsigned int camNum) + { + int result = 0; + + log("Printing device information for camera " + std::to_string(camNum) + "...\n", Priority::High); + + Spinnaker::GenApi::FeatureList_t features; + Spinnaker::GenApi::CCategoryPtr cCategoryPtr = iNodeMap.GetNode("DeviceInformation"); + if (Spinnaker::GenApi::IsAvailable(cCategoryPtr) && Spinnaker::GenApi::IsReadable(cCategoryPtr)) + { + cCategoryPtr->GetFeatures(features); + + Spinnaker::GenApi::FeatureList_t::const_iterator it; + for (it = features.begin(); it != features.end(); ++it) + { + Spinnaker::GenApi::CNodePtr pfeatureNode = *it; + const auto cValuePtr = (Spinnaker::GenApi::CValuePtr)pfeatureNode; + log(pfeatureNode->GetName() + " : " + + (IsReadable(cValuePtr) ? cValuePtr->ToString() : "Node not readable"), Priority::High); + } + } + else + log("Device control information not available.", Priority::High); + log(" ", Priority::High); + + return result; + } + #endif + + struct WPointGrey::ImplWPointGrey + { + #ifdef BUILD_MODULE_3D + bool mInitialized; + Spinnaker::CameraList mCameraList; + Spinnaker::SystemPtr mSystemPtr; + + ImplWPointGrey() : + mInitialized{false} + { + } + #endif + }; + + WPointGrey::WPointGrey() + #ifdef BUILD_MODULE_3D + : upImpl{new ImplWPointGrey{}} + #endif + { + try + { + #ifndef BUILD_MODULE_3D + error("OpenPose must be compiled with `BUILD_MODULE_3D` in order to use this class.", + __LINE__, __FUNCTION__, __FILE__); + #endif + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + WPointGrey::~WPointGrey() + { + #ifdef BUILD_MODULE_3D + try + { + if (upImpl->mInitialized) + { + // End acquisition for each camera + // Notice that what is usually a one-step process is now two steps + // because of the additional step of selecting the camera. It is worth + // repeating that camera selection needs to be done once per loop. + // It is possible to interact with cameras through the camera list with + // GetByIndex(); this is an alternative to retrieving cameras as + // Spinnaker::CameraPtr objects that can be quick and easy for small tasks. + // + for (auto i = 0; i < upImpl->mCameraList.GetSize(); i++) + upImpl->mCameraList.GetByIndex(i)->EndAcquisition(); + + for (auto i = 0; i < upImpl->mCameraList.GetSize(); i++) + { + // Select camera + auto cameraPtr = upImpl->mCameraList.GetByIndex(i); + + // Retrieve GenICam nodemap + auto& iNodeMap = cameraPtr->GetNodeMap(); + + // // Disable chunk data + // result = disableChunkData(iNodeMap); + // // if (result < 0) + // // return result; + + // Reset trigger + auto result = resetTrigger(iNodeMap); + if (result < 0) + error("Error happened..." + std::to_string(result), __LINE__, __FUNCTION__, __FILE__); + + // Deinitialize each camera + // Each camera must be deinitialized separately by first + // selecting the camera and then deinitializing it. + cameraPtr->DeInit(); + } + + log("Completed. Releasing...", Priority::High); + + // Clear camera list before releasing upImpl->mSystemPtr + upImpl->mCameraList.Clear(); + + // Release upImpl->mSystemPtr + upImpl->mSystemPtr->ReleaseInstance(); + } + + log("Done! Exitting...", Priority::High); + } + catch (const Spinnaker::Exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + #endif + } + + void WPointGrey::initializationOnThread() + { + #ifdef BUILD_MODULE_3D + try + { + upImpl->mInitialized = true; + + // Print application build information + log(std::string{ "Application build date: " } + __DATE__ + " " + __TIME__, Priority::High); + + // Retrieve singleton reference to upImpl->mSystemPtr object + upImpl->mSystemPtr = Spinnaker::System::GetInstance(); + + // Retrieve list of cameras from the upImpl->mSystemPtr + upImpl->mCameraList = upImpl->mSystemPtr->GetCameras(); + + unsigned int numCameras = upImpl->mCameraList.GetSize(); + + log("Number of cameras detected: " + std::to_string(numCameras), Priority::High); + + // Finish if there are no cameras + if (numCameras == 0) + { + // Clear camera list before releasing upImpl->mSystemPtr + upImpl->mCameraList.Clear(); + + // Release upImpl->mSystemPtr + upImpl->mSystemPtr->ReleaseInstance(); + + log("Not enough cameras!\nPress Enter to exit...", Priority::High); + getchar(); + + error("No cameras detected.", __LINE__, __FUNCTION__, __FILE__); + } + log("Camera system initialized...", Priority::High); + + // + // Retrieve transport layer nodemaps and print device information for + // each camera + // + // *** NOTES *** + // This example retrieves information from the transport layer nodemap + // twice: once to print device information and once to grab the device + // serial number. Rather than caching the nodemap, each nodemap is + // retrieved both times as needed. + // + log("\n*** DEVICE INFORMATION ***\n", Priority::High); + + for (int i = 0; i < upImpl->mCameraList.GetSize(); i++) + { + // Select camera + auto cameraPtr = upImpl->mCameraList.GetByIndex(i); + + // Retrieve TL device nodemap + auto& iNodeMapTLDevice = cameraPtr->GetTLDeviceNodeMap(); + + // Print device information + auto result = printDeviceInfo(iNodeMapTLDevice, i); + if (result < 0) + error("Result > 0, error " + std::to_string(result) + " occurred...", + __LINE__, __FUNCTION__, __FILE__); + } + + for (auto i = 0; i < upImpl->mCameraList.GetSize(); i++) + { + // Select camera + auto cameraPtr = upImpl->mCameraList.GetByIndex(i); + + // Initialize each camera + // You may notice that the steps in this function have more loops with + // less steps per loop; this contrasts the acquireImages() function + // which has less loops but more steps per loop. This is done for + // demonstrative purposes as both work equally well. + // Later: Each camera needs to be deinitialized once all images have been + // acquired. + cameraPtr->Init(); + + // Retrieve GenICam nodemap + // auto& iNodeMap = cameraPtr->GetNodeMap(); + + // // Configure trigger + // result = configureTrigger(iNodeMap); + // if (result < 0) + // error("Result > 0, error " + std::to_string(result) + " occurred...", + // __LINE__, __FUNCTION__, __FILE__); + + // // Configure chunk data + // result = configureChunkData(iNodeMap); + // if (result < 0) + // return result; + + // Remove buffer --> Always get newest frame + Spinnaker::GenApi::INodeMap& snodeMap = cameraPtr->GetTLStreamNodeMap(); + Spinnaker::GenApi::CEnumerationPtr ptrBufferHandlingMode = snodeMap.GetNode("StreamBufferHandlingMode"); + if (!Spinnaker::GenApi::IsAvailable(ptrBufferHandlingMode) + || !Spinnaker::GenApi::IsWritable(ptrBufferHandlingMode)) + error("Unable to change buffer handling mode", __LINE__, __FUNCTION__, __FILE__); + + Spinnaker::GenApi::CEnumEntryPtr ptrBufferHandlingModeNewest = ptrBufferHandlingMode->GetEntryByName( + "NewestFirstOverwrite" + ); + if (!Spinnaker::GenApi::IsAvailable(ptrBufferHandlingModeNewest) + || !IsReadable(ptrBufferHandlingModeNewest)) + error("Unable to set buffer handling mode to newest (entry 'NewestFirstOverwrite' retrieval)." + " Aborting...", __LINE__, __FUNCTION__, __FILE__); + int64_t bufferHandlingModeNewest = ptrBufferHandlingModeNewest->GetValue(); + + ptrBufferHandlingMode->SetIntValue(bufferHandlingModeNewest); + } + + // Prepare each camera to acquire images + // + // *** NOTES *** + // For pseudo-simultaneous streaming, each camera is prepared as if it + // were just one, but in a loop. Notice that cameras are selected with + // an index. We demonstrate pseduo-simultaneous streaming because true + // simultaneous streaming would require multiple process or threads, + // which is too complex for an example. + // + // Serial numbers are the only persistent objects we gather in this + // example, which is why a std::vector is created. + std::vector strSerialNumbers(upImpl->mCameraList.GetSize()); + for (auto i = 0u; i < strSerialNumbers.size(); i++) + { + // Select camera + auto cameraPtr = upImpl->mCameraList.GetByIndex(i); + + // Set acquisition mode to continuous + Spinnaker::GenApi::CEnumerationPtr ptrAcquisitionMode = cameraPtr->GetNodeMap().GetNode("AcquisitionMode"); + if (!Spinnaker::GenApi::IsAvailable(ptrAcquisitionMode) + || !Spinnaker::GenApi::IsWritable(ptrAcquisitionMode)) + error("Unable to set acquisition mode to continuous (node retrieval; camera " + std::to_string(i) + + "). Aborting...", __LINE__, __FUNCTION__, __FILE__); + + Spinnaker::GenApi::CEnumEntryPtr ptrAcquisitionModeContinuous = ptrAcquisitionMode->GetEntryByName( + "Continuous" + ); + if (!Spinnaker::GenApi::IsAvailable(ptrAcquisitionModeContinuous) + || !Spinnaker::GenApi::IsReadable(ptrAcquisitionModeContinuous)) + error("Unable to set acquisition mode to continuous (entry 'continuous' retrieval " + + std::to_string(i) + "). Aborting...", __LINE__, __FUNCTION__, __FILE__); + + int64_t acquisitionModeContinuous = ptrAcquisitionModeContinuous->GetValue(); + + ptrAcquisitionMode->SetIntValue(acquisitionModeContinuous); + + log("Camera " + std::to_string(i) + " acquisition mode set to continuous...", Priority::High); + + // Begin acquiring images + cameraPtr->BeginAcquisition(); + + log("Camera " + std::to_string(i) + " started acquiring images...", Priority::High); + + // Retrieve device serial number for filename + strSerialNumbers[i] = ""; + + Spinnaker::GenApi::CStringPtr ptrStringSerial = cameraPtr->GetTLDeviceNodeMap().GetNode( + "DeviceSerialNumber" + ); + + if (Spinnaker::GenApi::IsAvailable(ptrStringSerial) && Spinnaker::GenApi::IsReadable(ptrStringSerial)) + { + strSerialNumbers[i] = ptrStringSerial->GetValue(); + log("Camera " + std::to_string(i) + " serial number set to " + + strSerialNumbers[i].c_str() + "...", Priority::High); + } + log(" ", Priority::High); + } + + log("\nRunning for all cameras...\n\n*** IMAGE ACQUISITION ***\n", Priority::High); + } + catch (const Spinnaker::Exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + #endif + } + + std::shared_ptr> WPointGrey::workProducer() + { + try + { + #ifdef BUILD_MODULE_3D + try + { + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + // Get image from each camera + const auto cvMats = acquireImages(upImpl->mCameraList); + // Images to userDatum + auto datums3d = std::make_shared>(cvMats.size()); + for (auto i = 0u ; i < cvMats.size() ; i++) + datums3d->at(i).cvInputData = cvMats.at(i); + // Profiling speed + if (!cvMats.empty()) + { + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + } + // Return Datum + return datums3d; + } + catch (const Spinnaker::Exception& e) + { + this->stop(); + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return nullptr; + } + #else + error("OpenPose must be compiled with `BUILD_MODULE_3D` in order to use this class.", + __LINE__, __FUNCTION__, __FILE__); + return nullptr; + #endif + } + catch (const std::exception& e) + { + this->stop(); + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return nullptr; + } + } +} diff --git a/src/openpose/experimental/3d/reconstruction3D.cpp b/src/openpose/experimental/3d/reconstruction3D.cpp new file mode 100644 index 00000000..cb31bdf1 --- /dev/null +++ b/src/openpose/experimental/3d/reconstruction3D.cpp @@ -0,0 +1,181 @@ +#include +#include +#include + +namespace op +{ + double calcReprojectionError(const cv::Mat& X, const std::vector& M, const std::vector& pt2D) + { + auto averageError = 0.; + for(unsigned int i = 0 ; i < M.size() ; i++) + { + cv::Mat imageX = M[i] * X; + imageX /= imageX.at(2,0); + const auto error = std::sqrt(std::pow(imageX.at(0,0) - pt2D[i].x,2) + + std::pow(imageX.at(1,0) - pt2D[i].y,2)); + //log("Error: " + std::to_string(error)); + averageError += error; + } + return averageError / M.size(); + } + + void triangulate(cv::Mat& X, const std::vector& matrixEachCamera, + const std::vector& pointOnEachCamera) + { + // Security checks + if (matrixEachCamera.empty() || matrixEachCamera.size() != pointOnEachCamera.size()) + error("numberCameras.empty() || numberCameras.size() != pointOnEachCamera.size()", + __LINE__, __FUNCTION__, __FILE__); + // Create and fill A + const auto numberCameras = (int)matrixEachCamera.size(); + cv::Mat A = cv::Mat::zeros(numberCameras*2, 4, CV_64F); + for (auto i = 0 ; i < numberCameras ; i++) + { + cv::Mat temp = pointOnEachCamera[i].x*matrixEachCamera[i].rowRange(2,3) - matrixEachCamera[i].rowRange(0,1); + temp.copyTo(A.rowRange(i*2,i*2+1)); + temp = pointOnEachCamera[i].y*matrixEachCamera[i].rowRange(2,3) - matrixEachCamera[i].rowRange(1,2); + temp.copyTo(A.rowRange(i*2+1,i*2+2)); + } + // SVD on A + cv::SVD svd{A}; + svd.solveZ(A,X); + X /= X.at(3); + } + + // TODO: ask Hanbyul for the missing function: TriangulationOptimization + double triangulateWithOptimization(cv::Mat& X, const std::vector& matrixEachCamera, + const std::vector& pointOnEachCamera) + { + triangulate(X, matrixEachCamera, pointOnEachCamera); + + // //if (matrixEachCamera.size() >= 3) + // //double beforeError = calcReprojectionError(&matrixEachCamera, pointOnEachCamera, X); + // double change = TriangulationOptimization(&matrixEachCamera, pointOnEachCamera, X); + // //double afterError = calcReprojectionError(&matrixEachCamera,pointOnEachCamera,X); + // //printfLog("!!Mine %.8f , inFunc %.8f \n",beforeError-afterError,change); + // return change; + return 0.; + } + + void reconstructArray(Array& keypoints3D, const std::vector>& keypointsVector) + { + // Get number body parts + auto detectionMissed = false; + for (auto& keypoints : keypointsVector) + { + if (keypoints.empty()) + { + detectionMissed = true; + break; + } + } + // If at least one keypoints element not empty + if (!detectionMissed) + { + const auto numberBodyParts = keypointsVector.at(0).getSize(1); + // Create x-y vector from high score results + const auto threshold = 0.2f; + std::vector indexesUsed; + std::vector> xyPoints; + for (auto part = 0; part < numberBodyParts; part++) + { + // Create vector of points + auto missedPoint = false; + std::vector xyPointsElement; + for (auto& keypoints : keypointsVector) + { + if (keypoints[{0, part, 2}] > threshold) + xyPointsElement.emplace_back(cv::Point2d{ keypoints[{0, part, 0}], keypoints[{0, part, 1}]}); + else + { + missedPoint = true; + break; + } + } + if (!missedPoint) + { + indexesUsed.emplace_back(part); + xyPoints.emplace_back(xyPointsElement); + } + } + // 3D reconstruction + if (!xyPoints.empty()) + { + // Do 3D reconstruction + std::vector xyzPoints(xyPoints.size()); + for (auto i = 0u; i < xyPoints.size(); i++) + { + cv::Mat X; + triangulateWithOptimization(X, getMs(), xyPoints[i]); + xyzPoints[i] = cv::Point3d{ X.at(0), X.at(1), X.at(2) }; + } + + // 3D points to pose + // OpenCV alternative: + // http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#triangulatepoints + // cv::Mat reconstructedPoints{4, firstcv::Points.size(), CV_64F}; + // cv::triangulatecv::Points(cv::Mat::eye(3,4, CV_64F), M_3_1, firstcv::Points, secondcv::Points, + // reconstructedcv::Points); + keypoints3D = Array{ { 1, numberBodyParts, 4 }, 0 }; + for (auto index = 0u; index < indexesUsed.size(); index++) + { + auto& xValue = keypoints3D[{0, indexesUsed[index], 0}]; + auto& yValue = keypoints3D[{0, indexesUsed[index], 1}]; + auto& zValue = keypoints3D[{0, indexesUsed[index], 2}]; + auto& scoreValue = keypoints3D[{0, indexesUsed[index], 3}]; + if (std::isfinite(xyzPoints[index].x) && std::isfinite(xyzPoints[index].y) + && std::isfinite(xyzPoints[index].z)) + { + xValue = xyzPoints[index].x; + yValue = xyzPoints[index].y; + zValue = xyzPoints[index].z; + scoreValue = 1.f; + } + } + } + } + } + + void WReconstruction3D::work(std::shared_ptr>& datumsPtr) + { + // User's post-processing (after OpenPose processing & before OpenPose outputs) here + // datum.cvOutputData: rendered frame with pose or heatmaps + // datum.poseKeypoints: Array with the estimated pose + try + { + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + if (datumsPtr != nullptr && !datumsPtr->empty()) + { + std::vector> poseKeypointVector; + std::vector> faceKeypointVector; + std::vector> leftHandKeypointVector; + std::vector> rightHandKeypointVector; + for (auto& datumsElement : *datumsPtr) + { + poseKeypointVector.emplace_back(datumsElement.poseKeypoints); + faceKeypointVector.emplace_back(datumsElement.faceKeypoints); + leftHandKeypointVector.emplace_back(datumsElement.handKeypoints[0]); + rightHandKeypointVector.emplace_back(datumsElement.handKeypoints[1]); + } + // Pose 3-D reconstruction + reconstructArray(datumsPtr->at(0).poseKeypoints3D, poseKeypointVector); + // Face 3-D reconstruction + reconstructArray(datumsPtr->at(0).faceKeypoints3D, faceKeypointVector); + // Left hand 3-D reconstruction + reconstructArray(datumsPtr->at(0).leftHandKeypoints3D, leftHandKeypointVector); + // Right hand 3-D reconstruction + reconstructArray(datumsPtr->at(0).rightHandKeypoints3D, rightHandKeypointVector); + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + } + } + catch (const std::exception& e) + { + log("Some kind of unexpected error happened."); + this->stop(); + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } +} diff --git a/src/openpose/experimental/3d/renderer.cpp b/src/openpose/experimental/3d/renderer.cpp new file mode 100644 index 00000000..8baebe35 --- /dev/null +++ b/src/openpose/experimental/3d/renderer.cpp @@ -0,0 +1,464 @@ +#include +#include +#ifdef BUILD_MODULE_3D + #include +#endif +#include +#include +#include +#include +#include + +namespace op +{ + const bool LOG_VERBOSE_3D_RENDERER = false; + + #ifdef BUILD_MODULE_3D + struct Keypoints3D + { + Array mPoseKeypoints; + Array mFaceKeypoints; + Array mLeftHandKeypoints; + Array mRightHandKeypoints; + bool validKeypoints; + std::mutex mutex; + }; + + enum class CameraMode { + CAM_DEFAULT, + CAM_ROTATE, + CAM_PAN, + CAM_PAN_Z + }; + + Keypoints3D gKeypoints3D; + PoseModel sPoseModel = PoseModel::COCO_18; + + CameraMode gCameraMode = CameraMode::CAM_DEFAULT; + + const std::vector LIGHT_DIFFUSE{ 1.f, 1.f, 1.f, 1.f }; // Diffuse light + const std::vector LIGHT_POSITION{ 1.f, 1.f, 1.f, 0.f }; // Infinite light location + const std::vector COLOR_DIFFUSE{ 0.5f, 0.5f, 0.5f, 1.f }; + const std::string GUI_NAME{"OpenPose 3-D Reconstruction"}; + + const auto RAD_TO_DEG = 0.0174532925199433; + + //View Change by Mouse + bool gBButton1Down = false; + auto gXClick = 0.f; + auto gYClick = 0.f; + auto gGViewDistance = -250.f; // -82.3994f; //-45; + auto gMouseXRotate = -915.f; // -63.2f; //0; + auto gMouseYRotate = -5.f; // 7.f; //60; + auto gMouseXPan = -70.f; // 0; + auto gMouseYPan = -30.f; // 0; + auto gMouseZPan = 0.f; + auto gScaleForMouseMotion = 0.1f; + + void drawConeByTwoPts(const cv::Point3f& pt1, const cv::Point3f& pt2, const float ptSize) + { + const GLdouble x1 = pt1.x; + const GLdouble y1 = pt1.y; + const GLdouble z1 = pt1.z; + const GLdouble x2 = pt2.x; + const GLdouble y2 = pt2.y; + const GLdouble z2 = pt2.z; + + const double x = x2 - x1; + const double y = y2 - y1; + const double z = z2 - z1; + + glPushMatrix(); + + glTranslated(x1, y1, z1); + + if ((x != 0.) || (y != 0.)) + { + glRotated(std::atan2(y, x) / RAD_TO_DEG, 0., 0., 1.); + glRotated(std::atan2(std::sqrt(x*x + y*y), z) / RAD_TO_DEG, 0., 1., 0.); + } + else if (z<0) + glRotated(180, 1., 0., 0.); + + const auto height = std::sqrt((pt1.x - pt2.x)*(pt1.x - pt2.x) + (pt1.y - pt2.y)*(pt1.y - pt2.y) + + (pt1.z - pt2.z)*(pt1.z - pt2.z)); + glutSolidCone(ptSize, height, 5, 5); + + glPopMatrix(); + } + + void renderHumanBody(const Array& keypoints, const std::vector& pairs, + const std::vector colors, const float ratio) + { + const auto person = 0; + const auto numberPeople = keypoints.getSize(0); + const auto numberBodyParts = keypoints.getSize(1); + const auto numberColors = colors.size(); + const auto xOffset = -3000; // 640.f; + const auto yOffset = 360.f; + const auto zOffset = 1000; // 360.f; + const auto xScale = 43.f; + const auto yScale = 24.f; + const auto zScale = 24.f; + + if (numberPeople > person) + //for(int person=0;person keypointColor{ + colors[colorIndex % numberColors] / 255.f, + colors[(colorIndex + 1) % numberColors] / 255.f, + colors[(colorIndex + 2) % numberColors] / 255.f, + 1.f + }; + glMaterialfv(GL_FRONT_AND_BACK, GL_DIFFUSE, COLOR_DIFFUSE.data()); + glMaterialfv(GL_FRONT_AND_BACK, GL_AMBIENT, keypointColor.data()); + // Draw circle + const auto baseIndex = 4 * part + person*numberBodyParts; + if (keypoints[baseIndex + 3] > 0) + { + cv::Point3f keypoint{ + -(keypoints[baseIndex] - xOffset) / xScale, + -(keypoints[baseIndex + 1] - yOffset) / yScale, + (keypoints[baseIndex + 2] - zOffset) / zScale + }; + // Create and add new sphere + glPushMatrix(); + glTranslatef(keypoint.x, keypoint.y, keypoint.z); + // Draw sphere + glutSolidSphere(0.5 * ratio, 20, 20); + glPopMatrix(); + } + } + + // Lines connecting each keypoint pair + for (auto pair = 0u; pair < pairs.size(); pair += 2) + { + // Set color + const auto colorIndex = pairs[pair+1] * 3; + const std::vector keypointColor{ + colors[colorIndex % numberColors] / 255.f, + colors[(colorIndex + 1) % numberColors] / 255.f, + colors[(colorIndex + 2) % numberColors] / 255.f, + 1.f + }; + glMaterialfv(GL_FRONT_AND_BACK, GL_DIFFUSE, COLOR_DIFFUSE.data()); + glMaterialfv(GL_FRONT_AND_BACK, GL_AMBIENT, keypointColor.data()); + // Draw line + const auto baseIndexPairA = 4 * pairs[pair] + person*numberBodyParts; + const auto baseIndexPairB = 4 * pairs[pair + 1] + person*numberBodyParts; + if (keypoints[baseIndexPairA + 3] > 0 && keypoints[baseIndexPairB + 3] > 0) + { + cv::Point3f pairKeypointA{ + -(keypoints[baseIndexPairA] - xOffset) / xScale, + -(keypoints[baseIndexPairA + 1] - yOffset) / yScale, + (keypoints[baseIndexPairA + 2] - zOffset) / zScale + }; + cv::Point3f pairKeypointB{ + -(keypoints[baseIndexPairB] - xOffset) / xScale, + -(keypoints[baseIndexPairB + 1] - yOffset) / yScale, + (keypoints[baseIndexPairB + 2] - zOffset) / zScale + }; + drawConeByTwoPts(pairKeypointA, pairKeypointB, 0.5f * ratio); + } + } + } + } + + void initGraphics(void) + { + // Enable a single OpenGL light + glLightfv(GL_LIGHT0, GL_AMBIENT, LIGHT_DIFFUSE.data()); + glLightfv(GL_LIGHT0, GL_DIFFUSE, LIGHT_DIFFUSE.data()); + glLightfv(GL_LIGHT0, GL_POSITION, LIGHT_POSITION.data()); + glEnable(GL_LIGHT0); + glEnable(GL_LIGHTING); + + // Use depth buffering for hidden surface elimination + glEnable(GL_DEPTH_TEST); + + // Setup the view of the cube + glMatrixMode(GL_PROJECTION); + gluPerspective( /* field of view in degree */ 40.0, + /* aspect ratio */ 1.0, + /* Z near */ 1.0, /* Z far */ 1000.0); + glMatrixMode(GL_MODELVIEW); + gluLookAt( + 0.0, 0.0, 5.0, // eye is at (0,0,5) + 0.0, 0.0, 0.0, // center is at (0,0,0) + 0.0, 1.0, 0. // up is in positive Y direction + ); + + // Adjust cube position to be asthetic angle + glTranslatef(0.0, 0.0, -1.0); + glRotatef(60, 1.0, 0.0, 0.0); + glRotatef(-20, 0.0, 0.0, 1.0); + + glColorMaterial(GL_FRONT, GL_DIFFUSE); + glEnable(GL_COLOR_MATERIAL); + } + + // this is the actual idle function + void idleFunc() + { + glutPostRedisplay(); + glutSwapBuffers(); + } + + void renderFloor() + { + glDisable(GL_LIGHTING); + + const cv::Point3f gGloorCenter{ 0,0,0 }; //ankle + const cv::Point3f Noise{ 0,1,0 }; + + cv::Point3f upright = Noise - gGloorCenter; + upright = 1.0 / sqrt(upright.x *upright.x + upright.y *upright.y + upright.z *upright.z)*upright; + const cv::Point3f gGloorAxis2 = cv::Point3f{ 1,0,0 }.cross(upright); + const cv::Point3f gGloorAxis1 = gGloorAxis2.cross(upright); + + const auto gridNum = 10; + const auto width = 50.;//sqrt(Distance(gGloorPts.front(),gGloorCenter)*2 /gridNum) * 1.2; + const cv::Point3f origin = gGloorCenter - gGloorAxis1*(width*gridNum / 2) - gGloorAxis2*(width*gridNum / 2); + const cv::Point3f axis1 = gGloorAxis1 * width; + const cv::Point3f axis2 = gGloorAxis2 * width; + for (auto y = 0; y <= gridNum; ++y) + { + for (auto x = 0; x <= gridNum; ++x) + { + if ((x + y) % 2 == 0) + glColor4f(0.2f, 0.2f, 0.2f, 1.f); //black + else + glColor4f(0.5f, 0.5f, 0.5f, 1.f); //grey + + const cv::Point3f p1 = origin + axis1*x + axis2*y; + const cv::Point3f p2 = p1 + axis1; + const cv::Point3f p3 = p1 + axis2; + const cv::Point3f p4 = p1 + axis1 + axis2; + + glBegin(GL_QUADS); + + glVertex3f(p1.x, p1.y, p1.z); + glVertex3f(p2.x, p2.y, p2.z); + glVertex3f(p4.x, p4.y, p4.z); + glVertex3f(p3.x, p3.y, p3.z); + glEnd(); + } + } + glEnable(GL_LIGHTING); + } + + void renderMain(void) + { + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + + glLoadIdentity(); + //gluLookAt(0,0,0, 0, 0, 1, 0, -1, 0); + gluLookAt( + 0.0, 0.0, 5.0, // eye is at (0,0,5) + 0.0, 0.0, 0.0, // center is at (0,0,0) + 0.0, 1.0, 0. // up is in positive Y direction + ); + + glTranslatef(0, 0, gGViewDistance); + glRotatef(-gMouseYRotate, 1.f, 0.f, 0.f); + glRotatef(-gMouseXRotate, 0.f, 1.f, 0.f); + + glTranslatef(-gMouseXPan, gMouseYPan, -gMouseZPan); + + renderFloor(); + std::unique_lock lock{gKeypoints3D.mutex}; + if (gKeypoints3D.validKeypoints) + { + renderHumanBody(gKeypoints3D.mPoseKeypoints, getPoseBodyPartPairsRender(sPoseModel), + getPoseColors(sPoseModel), 1.f); + renderHumanBody(gKeypoints3D.mFaceKeypoints, FACE_PAIRS_RENDER, FACE_COLORS_RENDER, 0.5f); + renderHumanBody(gKeypoints3D.mLeftHandKeypoints, HAND_PAIRS_RENDER, HAND_COLORS_RENDER, 0.5f); + renderHumanBody(gKeypoints3D.mRightHandKeypoints, HAND_PAIRS_RENDER, HAND_COLORS_RENDER, 0.5f); + } + lock.unlock(); + + glutSwapBuffers(); + } + + void mouseButton(const int button, const int state, const int x, const int y) + { + + if (button == 3 || button == 4) //mouse wheel + { + if (button == 3) //zoom in + gGViewDistance += 10 * gScaleForMouseMotion; + else //zoom out + gGViewDistance -= 10 * gScaleForMouseMotion; + if (LOG_VERBOSE_3D_RENDERER) + log("gGViewDistance: " + std::to_string(gGViewDistance)); + } + else + { + if (button == GLUT_LEFT_BUTTON) + { + gBButton1Down = (state == GLUT_DOWN) ? 1 : 0; + gXClick = (float)x; + gYClick = (float)y; + + if (glutGetModifiers() == GLUT_ACTIVE_SHIFT) + gCameraMode = CameraMode::CAM_PAN; + else + gCameraMode = CameraMode::CAM_ROTATE; + } + if (LOG_VERBOSE_3D_RENDERER) + log("Clicked: [" + std::to_string(gXClick) + "," + std::to_string(gYClick) + "]"); + } + glutPostRedisplay(); + } + + void mouseMotion(const int x, const int y) + { + + // If button1 pressed, zoom in/out if mouse is moved up/down. + if (gBButton1Down) + { + if (gCameraMode == CameraMode::CAM_ROTATE) + { + gMouseXRotate += (x - gXClick)*0.2f; + gMouseYRotate -= (y - gYClick)*0.2f; + } + else if (gCameraMode == CameraMode::CAM_PAN) + { + gMouseXPan -= (x - gXClick) / 2 * gScaleForMouseMotion; + gMouseYPan -= (y - gYClick) / 2 * gScaleForMouseMotion; + } + else if (gCameraMode == CameraMode::CAM_PAN_Z) + { + auto dist = std::sqrt(pow((x - gXClick), 2.0f) + pow((y - gYClick), 2.0f)); + if (y < gYClick) + dist *= -1; + gMouseZPan -= dist / 5 * gScaleForMouseMotion; + } + + gXClick = (float)x; + gYClick = (float)y; + + glutPostRedisplay(); + if (LOG_VERBOSE_3D_RENDERER) + { + log("gMouseXRotate = " + std::to_string(gMouseXRotate)); + log("gMouseYRotate = " + std::to_string(gMouseYRotate)); + log("gMouseXPan = " + std::to_string(gMouseXPan)); + log("gMouseYPan = " + std::to_string(gMouseYPan)); + log("gMouseZPan = " + std::to_string(gMouseZPan)); + } + } + } + #endif + + WRender3D::WRender3D(const PoseModel poseModel) + { + try + { + #ifdef BUILD_MODULE_3D + // Update sPoseModel + sPoseModel = poseModel; + // Init display + cv::imshow(GUI_NAME, cv::Mat( 500, 500, CV_8UC3, cv::Scalar{ 0,0,0 } )); + //Run OpenGL + mRenderThread = std::thread{ &WRender3D::visualizationThread, this }; + #else + UNUSED(poseModel); + error("OpenPose must be compiled with `BUILD_MODULE_3D` in order to use this class.", + __LINE__, __FUNCTION__, __FILE__); + #endif + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void WRender3D::workConsumer(const std::shared_ptr>& datumsPtr) + { + #ifdef BUILD_MODULE_3D + try + { + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + // User's displaying/saving/other processing here + // datum.cvOutputData: rendered frame with pose or heatmaps + // datum.poseKeypoints: Array with the estimated pose + if (datumsPtr != nullptr && !datumsPtr->empty()) + { + cv::Mat cvMat = datumsPtr->at(0).cvOutputData.clone(); + for (auto i = 1u; i < datumsPtr->size(); i++) + cv::hconcat(cvMat, datumsPtr->at(i).cvOutputData, cvMat); + // while (cvMat.cols > 1500 || cvMat.rows > 1500) + while (cvMat.cols > 1920 || cvMat.rows > 1920) + // while (cvMat.rows > 3500) + cv::pyrDown(cvMat, cvMat); + // Display all views + cv::imshow(GUI_NAME, cvMat); + cv::resizeWindow(GUI_NAME, cvMat.cols, cvMat.rows); + // OpenGL Rendering + std::unique_lock lock{gKeypoints3D.mutex}; + gKeypoints3D.mPoseKeypoints = datumsPtr->at(0).poseKeypoints3D; + gKeypoints3D.mFaceKeypoints = datumsPtr->at(0).faceKeypoints3D; + gKeypoints3D.mLeftHandKeypoints = datumsPtr->at(0).leftHandKeypoints3D; + gKeypoints3D.mRightHandKeypoints = datumsPtr->at(0).rightHandKeypoints3D; + gKeypoints3D.validKeypoints = true; + lock.unlock(); + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + } + // Render images + // It sleeps 1 ms just to let the user see the output. Change to 33ms for normal 30 fps display if too fast + cv::waitKey(1); + } + catch (const std::exception& e) + { + log("Some kind of unexpected error happened."); + this->stop(); + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + #else + UNUSED(datumsPtr); + #endif + } + + void WRender3D::visualizationThread() + { + #ifdef BUILD_MODULE_3D + try + { + char *my_argv[] = { NULL }; + int my_argc = 0; + glutInit(&my_argc, my_argv); + + // setup the size, position, and display mode for new windows + glutInitWindowSize(1280, 720); + glutInitWindowPosition(200, 0); + // glutSetOption(GLUT_MULTISAMPLE,8); + glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH | GLUT_MULTISAMPLE); + + // create and set up a window + glutCreateWindow(GUI_NAME.c_str()); + initGraphics(); + glutDisplayFunc(renderMain); + glutMouseFunc(mouseButton); + glutMotionFunc(mouseMotion); + glutIdleFunc(idleFunc); + + glutMainLoop(); + + this->stop(); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + #endif + } +} diff --git a/src/openpose/experimental/CMakeLists.txt b/src/openpose/experimental/CMakeLists.txt index 98ed60d8..ca8bdb45 100644 --- a/src/openpose/experimental/CMakeLists.txt +++ b/src/openpose/experimental/CMakeLists.txt @@ -1,8 +1,9 @@ +add_subdirectory(3d) add_subdirectory(tracking) if (UNIX AND NOT APPLE) add_custom_target(openpose_experimental) - add_dependencies(openpose_experimental openpose_tracking) + add_dependencies(openpose_experimental openpose_tracking openpose_3d) endif (UNIX AND NOT APPLE) -set(SOURCES_OPENPOSE ${SOURCES_OPENPOSE} PARENT_SCOPE) \ No newline at end of file +set(SOURCES_OPENPOSE ${SOURCES_OPENPOSE} PARENT_SCOPE) diff --git a/src/openpose/experimental/tracking/CMakeLists.txt b/src/openpose/experimental/tracking/CMakeLists.txt index 9ef519b3..a9b79d8f 100644 --- a/src/openpose/experimental/tracking/CMakeLists.txt +++ b/src/openpose/experimental/tracking/CMakeLists.txt @@ -17,4 +17,4 @@ if (UNIX AND NOT APPLE) RUNTIME DESTINATION bin LIBRARY DESTINATION lib ARCHIVE DESTINATION lib/openpose) -endif (UNIX AND NOT APPLE) \ No newline at end of file +endif (UNIX AND NOT APPLE) diff --git a/ubuntu/Makefile.config.Ubuntu14_cuda7.example b/ubuntu/Makefile.config.Ubuntu14_cuda7.example index 0e5b5706..43128b2c 100644 --- a/ubuntu/Makefile.config.Ubuntu14_cuda7.example +++ b/ubuntu/Makefile.config.Ubuntu14_cuda7.example @@ -52,6 +52,11 @@ DEEP_NET := caffe # Caffe directory CAFFE_DIR := 3rdparty/caffe/distribute +# OpenPose 3-D Reconstruction +# BUILD_MODULE_3D := 1 +# Spinnaker directory +SPINNAKER_DIR := /usr/include/spinnaker + # Whatever else you find you need goes here. INCLUDE_DIRS := /usr/local/include LIBRARY_DIRS := /usr/local/lib /usr/lib diff --git a/ubuntu/Makefile.config.Ubuntu14_cuda8.example b/ubuntu/Makefile.config.Ubuntu14_cuda8.example index 608c1c3a..cb448414 100644 --- a/ubuntu/Makefile.config.Ubuntu14_cuda8.example +++ b/ubuntu/Makefile.config.Ubuntu14_cuda8.example @@ -58,6 +58,11 @@ DEEP_NET := caffe # Caffe directory CAFFE_DIR := 3rdparty/caffe/distribute +# OpenPose 3-D Reconstruction +# BUILD_MODULE_3D := 1 +# Spinnaker directory +SPINNAKER_DIR := /usr/include/spinnaker + # Whatever else you find you need goes here. INCLUDE_DIRS := /usr/local/include LIBRARY_DIRS := /usr/local/lib /usr/lib diff --git a/ubuntu/Makefile.config.Ubuntu16_cuda7.example b/ubuntu/Makefile.config.Ubuntu16_cuda7.example index 25793854..3fbd60fb 100644 --- a/ubuntu/Makefile.config.Ubuntu16_cuda7.example +++ b/ubuntu/Makefile.config.Ubuntu16_cuda7.example @@ -52,6 +52,11 @@ DEEP_NET := caffe # Caffe directory CAFFE_DIR := 3rdparty/caffe/distribute +# OpenPose 3-D Reconstruction +# BUILD_MODULE_3D := 1 +# Spinnaker directory +SPINNAKER_DIR := /usr/include/spinnaker + # Whatever else you find you need goes here. INCLUDE_DIRS := /usr/local/include /usr/include/hdf5/serial LIBRARY_DIRS := /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/hdf5/serial diff --git a/ubuntu/Makefile.config.Ubuntu16_cuda8.example b/ubuntu/Makefile.config.Ubuntu16_cuda8.example index 233f2dfc..1830999f 100644 --- a/ubuntu/Makefile.config.Ubuntu16_cuda8.example +++ b/ubuntu/Makefile.config.Ubuntu16_cuda8.example @@ -58,6 +58,11 @@ DEEP_NET := caffe # Caffe directory CAFFE_DIR := 3rdparty/caffe/distribute +# OpenPose 3-D Reconstruction +# BUILD_MODULE_3D := 1 +# Spinnaker directory +SPINNAKER_DIR := /usr/include/spinnaker + # Whatever else you find you need goes here. INCLUDE_DIRS := /usr/local/include /usr/include/hdf5/serial LIBRARY_DIRS := /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/hdf5/serial diff --git a/ubuntu/Makefile.config.Ubuntu16_cuda8_JetsonTX2 b/ubuntu/Makefile.config.Ubuntu16_cuda8_JetsonTX2 index 476fc9f9..75563a59 100644 --- a/ubuntu/Makefile.config.Ubuntu16_cuda8_JetsonTX2 +++ b/ubuntu/Makefile.config.Ubuntu16_cuda8_JetsonTX2 @@ -58,6 +58,11 @@ DEEP_NET := caffe # Caffe directory CAFFE_DIR := 3rdparty/caffe/distribute +# OpenPose 3-D Reconstruction +# BUILD_MODULE_3D := 1 +# Spinnaker directory +SPINNAKER_DIR := /usr/include/spinnaker + # Whatever else you find you need goes here. INCLUDE_DIRS := /usr/local/include /usr/include/hdf5/serial LIBRARY_DIRS := /usr/local/lib /usr/lib /usr/lib/aarch64-linux-gnu /usr/lib/aarch64-linux-gnu/hdf5/serial diff --git a/ubuntu/Makefile.example b/ubuntu/Makefile.example index 65b6d89d..c3f991f0 100644 --- a/ubuntu/Makefile.example +++ b/ubuntu/Makefile.example @@ -46,8 +46,8 @@ COMMON_FLAGS += -DOPEN_POSE_VERSION=$(DYNAMIC_VERSION_MAJOR).$(DYNAMIC_VERSION_M # Enable profiler ############################## PROFILER_ENABLED ?= 0 -ifeq ($(PROFILER_ENABLED), 1) - COMMON_FLAGS += -DPROFILER_ENABLED +ifneq ($(PROFILER_ENABLED), 0) + COMMON_FLAGS += -DPROFILER_ENABLED=$(PROFILER_ENABLED) endif @@ -70,6 +70,19 @@ else LIBRARY_DIRS += $(CAFFE_DIR)/lib endif + +# Spinnaker +BUILD_MODULE_3D ?= 0 +ifneq ($(BUILD_MODULE_3D), 0) + COMMON_FLAGS += -DBUILD_MODULE_3D + # Spinnaker SDK + LIBRARIES += Spinnaker + INCLUDE_DIRS += $(SPINNAKER_DIR) + # FreeGLUT + LIBRARIES += glut + # LIBRARIES += GLU GL glut +endif + ############################## # Get all source files ############################## @@ -330,8 +343,6 @@ ifeq ($(USE_PKG_CONFIG), 1) else PKG_CONFIG := endif -# LDFLAGS += $(foreach librarydir,$(LIBRARY_DIRS),-L$(librarydir)) $(PKG_CONFIG) \ -# $(foreach library,$(LIBRARIES),-l$(library)) -Wl,-rpath=$(CAFFE_DIR)/lib LDFLAGS += $(foreach librarydir,$(LIBRARY_DIRS),-L$(librarydir)) $(PKG_CONFIG) \ $(foreach library,$(LIBRARIES),-l$(library)) diff --git a/windows/OpenPose/OpenPose.vcxproj b/windows/OpenPose/OpenPose.vcxproj index 4b1aa3c8..4a50d9c6 100644 --- a/windows/OpenPose/OpenPose.vcxproj +++ b/windows/OpenPose/OpenPose.vcxproj @@ -123,6 +123,12 @@ + + + + + + @@ -273,6 +279,10 @@ + + + + diff --git a/windows/OpenPose/OpenPose.vcxproj.filters b/windows/OpenPose/OpenPose.vcxproj.filters index 83ba5968..74d97533 100644 --- a/windows/OpenPose/OpenPose.vcxproj.filters +++ b/windows/OpenPose/OpenPose.vcxproj.filters @@ -84,6 +84,12 @@ {24e85903-dd30-4fe6-af6f-154724a96a70} + + {e6ceb1d2-9199-4dfa-922e-430b06e67330} + + + {080754ef-07d7-41ed-a363-509c1202f108} + @@ -545,6 +551,24 @@ Header Files\experimental\tracking + + Header Files\experimental\3d + + + Header Files\experimental\3d + + + Header Files\experimental\3d + + + Header Files\experimental\3d + + + Header Files\experimental\3d + + + Header Files\experimental\3d + @@ -807,6 +831,18 @@ Source Files\experimental\tracking + + Source Files\experimental\3d + + + Source Files\experimental\3d + + + Source Files\experimental\3d + + + Source Files\experimental\3d + diff --git a/windows/OpenPose3DReconstruction/OpenPose3DReconstruction.vcxproj b/windows/OpenPose3DReconstruction/OpenPose3DReconstruction.vcxproj index 8682f884..f9764c9c 100644 --- a/windows/OpenPose3DReconstruction/OpenPose3DReconstruction.vcxproj +++ b/windows/OpenPose3DReconstruction/OpenPose3DReconstruction.vcxproj @@ -11,18 +11,7 @@ - - - - - - - - - - - - + {F31BEEF2-37B0-4B0F-85D3-AD427C56AD72} diff --git a/windows/OpenPose3DReconstruction/OpenPose3DReconstruction.vcxproj.filters b/windows/OpenPose3DReconstruction/OpenPose3DReconstruction.vcxproj.filters index 698cbda7..0b9b0887 100644 --- a/windows/OpenPose3DReconstruction/OpenPose3DReconstruction.vcxproj.filters +++ b/windows/OpenPose3DReconstruction/OpenPose3DReconstruction.vcxproj.filters @@ -7,42 +7,10 @@ {52199589-120f-4517-bd6f-64c2857a0e29} - - {af413d97-7c26-410a-ad92-33f3904f4cde} - - + Source Files - - Source Files - - - Source Files - - - Source Files - - - - - Header Files\openpose3d - - - Header Files\openpose3d - - - Header Files\openpose3d - - - Header Files\openpose3d - - - Header Files\openpose3d - - - Header Files\openpose3d - \ No newline at end of file