From 66b02dbda39622317d600dd4fa2cbba0f02e51cf Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Thu, 7 Jun 2018 19:26:48 -0400 Subject: [PATCH] Calibration mode 3 improved --- doc/installation.md | 2 +- examples/calibration/calibration.cpp | 6 +- src/openpose/3d/poseTriangulation.cpp | 7 +- .../calibration/cameraParameterEstimation.cpp | 234 ++++++++++++------ src/openpose/wrapper/wrapperAuxiliary.cpp | 10 + 5 files changed, 174 insertions(+), 85 deletions(-) diff --git a/doc/installation.md b/doc/installation.md index cb4d3cec..cd0e1ad2 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -124,7 +124,7 @@ The instructions in this section describe the steps to build OpenPose using CMak - [Caffe dependencies](http://posefs1.perception.cs.cmu.edu/OpenPose/3rdparty/windows/caffe3rdparty_2017_07_14.zip): Unzip as `3rdparty/windows/caffe3rdparty/`. - [OpenCV 3.1](http://posefs1.perception.cs.cmu.edu/OpenPose/3rdparty/windows/opencv_310.zip): Unzip as `3rdparty/windows/opencv/`. 6. Mac - **Caffe, OpenCV, and Caffe prerequisites**: - - Install deps by running `sudo bash 3rdparty/osx/install_deps.sh` on your terminal. + - Install deps by running `bash 3rdparty/osx/install_deps.sh` on your terminal. - Current OSX has only been tested with the CPU Version, and hence must be compiled with the `-DGPU_MODE=CPU_ONLY` flag. 7. **Eigen prerequisite**: - Note: This step is optional, only required for some specific extra functionality, such as extrinsic camera calibration. diff --git a/examples/calibration/calibration.cpp b/examples/calibration/calibration.cpp index 2bed6e45..ff2742f5 100644 --- a/examples/calibration/calibration.cpp +++ b/examples/calibration/calibration.cpp @@ -24,7 +24,7 @@ DEFINE_int32(logging_level, 3, "The logging level. Inte " 255 will not output any. Current OpenPose library messages are in the range 0-4: 1 for" " low priority messages and 4 for important ones."); // Calibration -DEFINE_int32(mode, 1, "Select 1 for intrinsic camera parameter calibration, 2 for extrinsic ones."); +DEFINE_int32(mode, 1, "Select 1 for intrinsic camera parameter calibration, 2 for extrinsic calibration."); DEFINE_string(calibration_image_dir, "images/intrinsics/", "Directory where the images for camera parameter calibration are placed."); DEFINE_double(grid_square_size_mm, 127.0, "Chessboard square length (in millimeters)."); DEFINE_string(grid_number_inner_corners,"9x6", "Number of inner corners in width and height, i.e., number of total squares in width" @@ -95,7 +95,7 @@ int openPoseDemo() op::log("Extrinsic calibration completed!", op::Priority::High); } - // // Calibration - Extrinsics for Visual SFM + // // Calibration - Extrinsics Refinement with Visual SFM // else if (FLAGS_mode == 3) // { // op::log("Running calibration (intrinsic parameters)...", op::Priority::High); @@ -105,7 +105,7 @@ int openPoseDemo() // // const auto saveImagesWithCorners = true; // // Run camera calibration code // op::estimateAndSaveSiftFile(gridInnerCorners, - // op::formatAsDirectory(FLAGS_camera_parameter_folder), + // op::formatAsDirectory(FLAGS_calibration_image_dir), // FLAGS_number_cameras, // saveImagesWithCorners); // op::log("Intrinsic calibration completed!", op::Priority::High); diff --git a/src/openpose/3d/poseTriangulation.cpp b/src/openpose/3d/poseTriangulation.cpp index 2537f36e..1c31d7f4 100644 --- a/src/openpose/3d/poseTriangulation.cpp +++ b/src/openpose/3d/poseTriangulation.cpp @@ -306,6 +306,7 @@ namespace op } } // 3D reconstruction + auto reprojectionErrorTotal = 0.0; keypoints3D.reset({ 1, numberBodyParts, 4 }, 0); if (!xyPoints.empty()) { @@ -314,10 +315,13 @@ namespace op for (auto i = 0u; i < xyPoints.size(); i++) { cv::Mat reconstructedPoint; - triangulateWithOptimization(reconstructedPoint, cameraMatricesPerPoint[i], xyPoints[i]); + reprojectionErrorTotal += triangulateWithOptimization(reconstructedPoint, + cameraMatricesPerPoint[i], + xyPoints[i]); xyzPoints[i] = cv::Point3d{reconstructedPoint.at(0), reconstructedPoint.at(1), reconstructedPoint.at(2)}; } + reprojectionErrorTotal /= xyPoints.size(); // 3D points to pose // OpenCV alternative: @@ -339,6 +343,7 @@ namespace op } } } + // log("Reprojection error: " + std::to_string(reprojectionErrorTotal)); // To debug reprojection error } return keypoints3D; } diff --git a/src/openpose/calibration/cameraParameterEstimation.cpp b/src/openpose/calibration/cameraParameterEstimation.cpp index 41c81b7b..6cad6466 100644 --- a/src/openpose/calibration/cameraParameterEstimation.cpp +++ b/src/openpose/calibration/cameraParameterEstimation.cpp @@ -1,5 +1,6 @@ #include #include // std::accumulate +#include #include #ifdef WITH_EIGEN #include @@ -732,6 +733,127 @@ namespace op log("Cannot write on " + fileName, Priority::High); } + std::string getFileNameFromCameraIndex(const int cameraIndex) + { + try + { + // Security checks + if (cameraIndex >= 100) + error("Only implemented for up to 99 cameras.", __LINE__, __FUNCTION__, __FILE__); + // Return result + return (cameraIndex < 10 ? "00_0" : "00_") + std::to_string(cameraIndex); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return ""; + } + } + + void estimateAndSaveSiftFileSubThread(std::vector* points2DExtrinsicPtr, + std::vector* matchIndexesCameraPtr, + const int cameraIndex, + const int numberCameras, + const int numberCorners, + const unsigned int numberViews, + const bool saveImagesWithCorners, + const std::string& imagesFolder, + const cv::Size& gridInnerCornersCvSize, + const cv::Size& imageSize, + const std::vector>& imageAndPaths) + { + try + { + // Security checks + if (points2DExtrinsicPtr == nullptr || matchIndexesCameraPtr == nullptr) + error("Make sure than points2DExtrinsicPtr != nullptr && matchIndexesCameraPtr != nullptr.", + __LINE__, __FUNCTION__, __FILE__); + std::vector& points2DExtrinsic = *points2DExtrinsicPtr; + std::vector& matchIndexesCamera = *matchIndexesCameraPtr; + std::vector imagesWithCorners; + for (auto viewIndex = 0u ; viewIndex < numberViews ; viewIndex++) + { + // Get right image + const auto& imageAndPath = imageAndPaths.at(viewIndex * numberCameras + cameraIndex); + const auto& image = imageAndPath.first; + + if (viewIndex % std::max(1, int(numberViews/6)) == 0) + log("Camera " + std::to_string(cameraIndex) + " - Image view " + + std::to_string(viewIndex+1) + "/" + std::to_string(numberViews), + Priority::High); + + // Security check + if (imageSize.width != image.cols || imageSize.height != image.rows) + error("Detected images with different sizes in `" + imagesFolder + "` All images" + " must have the same resolution.", __LINE__, __FUNCTION__, __FILE__); + + // Find grid corners + bool found; + std::vector points2DVector; + std::tie(found, points2DVector) = findAccurateGridCorners(image, gridInnerCornersCvSize); + + // Reorder & save 2D pixels points + if (found) + { + reorderPoints(points2DVector, gridInnerCornersCvSize, Points2DOrigin::TopLeft); + for (auto i = 0 ; i < numberCorners ; i++) + matchIndexesCamera.emplace_back(viewIndex * numberCorners + i); + } + else + { + points2DVector.clear(); + points2DVector.resize(numberCorners, cv::Point2f{-1.f,-1.f}); + log("Camera " + std::to_string(cameraIndex) + " - Image view " + + std::to_string(viewIndex+1) + "/" + std::to_string(numberViews) + + " - Chessboard not found.", Priority::High); + } + points2DExtrinsic.insert(points2DExtrinsic.end(), points2DVector.begin(), points2DVector.end()); + + // Show image (with chessboard corners if found) + if (saveImagesWithCorners) + { + cv::Mat imageToPlot = image.clone(); + if (found) + drawGridCorners(imageToPlot, gridInnerCornersCvSize, points2DVector); + imagesWithCorners.emplace_back(imageToPlot); + } + } + + // Save *.sift file for camera + // const auto fileName = getFullFilePathNoExtension(imageAndPaths.at(cameraIndex).second) + ".sift"; + const auto fileName = getFileParentFolderPath(imageAndPaths.at(cameraIndex).second) + + getFileNameFromCameraIndex(cameraIndex) + ".sift"; + writeVisualSFMSiftGPU(fileName, points2DExtrinsic); + + // Save images with corners + if (saveImagesWithCorners) + { + const auto folderWhereSavingImages = imagesFolder + "images_with_corners/"; + // Create directory in case it did not exist + makeDirectory(folderWhereSavingImages); + const auto pathWhereSavingImages = folderWhereSavingImages + std::to_string(cameraIndex) + "_"; + // Save new images + const std::string extension{".png"}; + auto fileRemoved = true; + for (auto i = 0u ; i < imagesWithCorners.size() || fileRemoved; i++) + { + const auto finalPath = pathWhereSavingImages + std::to_string(i+1) + extension; + // remove leftovers/previous files + // Note: If file is not deleted before cv::imwrite, Windows considers that the file + // was "only" modified at that time, not created + fileRemoved = {remove(finalPath.c_str()) == 0}; + // save images on hhd in the desired place + if (i < imagesWithCorners.size()) + saveImage(imagesWithCorners.at(i), finalPath); + } + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + @@ -782,13 +904,16 @@ namespace op std::cerr << "Chessboard not found in this image." << std::endl; // Show image (with chessboard corners if found) - cv::Mat imageToPlot = image.clone(); - if (found) - drawGridCorners(imageToPlot, gridInnerCornersCvSize, points2DVector); - // cv::pyrDown(imageToPlot, imageToPlot); - // cv::imshow("Image View", imageToPlot); - // cv::waitKey(delayMilliseconds); - imagesWithCorners.emplace_back(imageToPlot); + if (saveImagesWithCorners) + { + cv::Mat imageToPlot = image.clone(); + if (found) + drawGridCorners(imageToPlot, gridInnerCornersCvSize, points2DVector); + // cv::pyrDown(imageToPlot, imageToPlot); + // cv::imshow("Image View", imageToPlot); + // cv::waitKey(delayMilliseconds); + imagesWithCorners.emplace_back(imageToPlot); + } } // Run calibration @@ -1083,58 +1208,27 @@ namespace op std::vector imagesWithCorners; const auto imageSize = imageAndPaths.at(0).first.size(); const auto numberViews = imageAndPaths.size() / numberCameras; + log("Processing cameras...", Priority::High); + std::vector threads; for (auto cameraIndex = 0 ; cameraIndex < numberCameras ; cameraIndex++) { - log("Processing camera " + std::to_string(cameraIndex) + "...", Priority::High); - auto& points2DExtrinsic = points2DVectorsExtrinsic[cameraIndex]; - auto& matchIndexesCamera = matchIndexes[cameraIndex]; - for (auto viewIndex = 0u ; viewIndex < numberViews ; viewIndex++) - { - // Get right image - const auto& imageAndPath = imageAndPaths.at(viewIndex * numberCameras + cameraIndex); - const auto& image = imageAndPath.first; - - if (viewIndex % std::max(1, int(numberViews/10)) == 0) - log("Image view " + std::to_string(viewIndex+1) + "/" + std::to_string(numberViews), - Priority::High); - - // Security check - if (imageSize.width != image.cols || imageSize.height != image.rows) - error("Detected images with different sizes in `" + imagesFolder + "` All images" - " must have the same resolution.", __LINE__, __FUNCTION__, __FILE__); - - // Find grid corners - bool found; - std::vector points2DVector; - std::tie(found, points2DVector) = findAccurateGridCorners(image, gridInnerCornersCvSize); - - // Reorder & save 2D pixels points - if (found) - { - reorderPoints(points2DVector, gridInnerCornersCvSize, Points2DOrigin::TopLeft); - for (auto i = 0 ; i < numberCorners ; i++) - matchIndexesCamera.emplace_back(viewIndex * numberCorners + i); - } - else - { - points2DVector.clear(); - points2DVector.resize(numberCorners, cv::Point2f{-1.f,-1.f}); - log("Chessboard not found in view " + std::to_string(viewIndex) - + ", camera " + std::to_string(cameraIndex) + ".", Priority::High); - } - points2DExtrinsic.insert(points2DExtrinsic.end(), points2DVector.begin(), points2DVector.end()); - - // Show image (with chessboard corners if found) - cv::Mat imageToPlot = image.clone(); - if (found) - drawGridCorners(imageToPlot, gridInnerCornersCvSize, points2DVector); - imagesWithCorners.emplace_back(imageToPlot); - } - - // Save *.sift file for camera - const auto fileName = getFullFilePathNoExtension(imageAndPaths.at(cameraIndex).second) + ".sift"; - writeVisualSFMSiftGPU(fileName, points2DExtrinsic); + auto* points2DExtrinsic = &points2DVectorsExtrinsic[cameraIndex]; + auto* matchIndexesCamera = &matchIndexes[cameraIndex]; + // Threaded version + threads.emplace_back(estimateAndSaveSiftFileSubThread, points2DExtrinsic, + matchIndexesCamera, cameraIndex, numberCameras, + numberCorners, numberViews, saveImagesWithCorners, imagesFolder, + gridInnerCornersCvSize, imageSize, imageAndPaths); + // // Non-threaded version + // estimateAndSaveSiftFileSubThread(points2DExtrinsic, matchIndexesCamera, cameraIndex, numberCameras, + // numberCorners, numberViews, saveImagesWithCorners, imagesFolder, + // gridInnerCornersCvSize, imageSize, imageAndPaths); } + // Threaded version + for (auto& thread : threads) + if (thread.joinable()) + thread.join(); + // Matching file std::ofstream ofstreamMatches{getFileParentFolderPath(imageAndPaths.at(0).second) + "FeatureMatches.txt"}; for (auto cameraIndex = 0 ; cameraIndex < numberCameras ; cameraIndex++) @@ -1146,8 +1240,10 @@ namespace op matchIndexes[cameraIndex2].begin(), matchIndexes[cameraIndex2].end(), std::back_inserter(matchIndexesIntersection)); - ofstreamMatches << getFileNameAndExtension(imageAndPaths.at(cameraIndex).second) - << " " << getFileNameAndExtension(imageAndPaths.at(cameraIndex2).second) + ofstreamMatches << getFileNameFromCameraIndex(cameraIndex) << ".jpg" + << " " << getFileNameFromCameraIndex(cameraIndex2) << ".jpg" + // ofstreamMatches << getFileNameAndExtension(imageAndPaths.at(cameraIndex).second) + // << " " << getFileNameAndExtension(imageAndPaths.at(cameraIndex2).second) << " " << matchIndexesIntersection.size() << "\n"; for (auto reps = 0 ; reps < 2 ; reps++) { @@ -1166,28 +1262,6 @@ namespace op for (auto i = 1 ; i < numberCameras ; i++) if (points2DVectorsExtrinsic[i].size() != points2DVectorsExtrinsic[0].size()) error("Something went wrong. Notify us.", __LINE__, __FUNCTION__, __FILE__); - - // Save images with corners - if (saveImagesWithCorners) - { - const auto folderWhereSavingImages = imagesFolder + "images_with_corners/"; - // Create directory in case it did not exist - makeDirectory(folderWhereSavingImages); - // Save new images - const std::string extension{".png"}; - auto fileRemoved = true; - for (auto i = 0u ; i < imagesWithCorners.size() || fileRemoved; i++) - { - const auto finalPath = folderWhereSavingImages + std::to_string(i+1) + extension; - // remove leftovers/previous files - // Note: If file is not deleted before cv::imwrite, Windows considers that the file - // was "only" modified at that time, not created - fileRemoved = {remove(finalPath.c_str()) == 0}; - // save images on hhd in the desired place - if (i < imagesWithCorners.size()) - saveImage(imagesWithCorners.at(i), finalPath); - } - } } catch (const std::exception& e) { diff --git a/src/openpose/wrapper/wrapperAuxiliary.cpp b/src/openpose/wrapper/wrapperAuxiliary.cpp index 51e46427..8dda65e5 100644 --- a/src/openpose/wrapper/wrapperAuxiliary.cpp +++ b/src/openpose/wrapper/wrapperAuxiliary.cpp @@ -61,7 +61,17 @@ namespace op || !wrapperStructOutput.writeCocoJson.empty() || !wrapperStructOutput.writeHeatMaps.empty() || !wrapperStructOutput.writeCocoFootJson.empty() ); + const auto savingCvOutput = ( + !wrapperStructOutput.writeImages.empty() || !wrapperStructOutput.writeVideo.empty() + ); const bool guiEnabled = (wrapperStructOutput.displayMode != DisplayMode::NoDisplay); + if (!guiEnabled && !savingCvOutput && renderOutput) + { + const auto message = "GUI is not enabled and you are not saving the output frames. You should then" + " disable rendering for a faster code. I.e., add `--render_pose 0`." + + additionalMessage; + error(message, __LINE__, __FUNCTION__, __FILE__); + } if (!guiEnabled && !savingSomething) { const auto message = "No output is selected (`--display 0`) and no results are generated (no"