Visual Servoing Platform  version 3.6.1 under development (2025-02-18)
testKeyPoint-4.cpp
1 /*
2  * ViSP, open source Visual Servoing Platform software.
3  * Copyright (C) 2005 - 2024 by Inria. All rights reserved.
4  *
5  * This software is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  * See the file LICENSE.txt at the root directory of this source
10  * distribution for additional information about the GNU GPL.
11  *
12  * For using ViSP with software that can not be combined with the GNU
13  * GPL, please contact Inria about acquiring a ViSP Professional
14  * Edition License.
15  *
16  * See https://visp.inria.fr for more information.
17  *
18  * This software was developed at:
19  * Inria Rennes - Bretagne Atlantique
20  * Campus Universitaire de Beaulieu
21  * 35042 Rennes Cedex
22  * France
23  *
24  * If you have questions regarding the use of this file, please contact
25  * Inria at visp@inria.fr
26  *
27  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
28  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
29  *
30  * Description:
31  * Test keypoint matching and pose estimation with mostly OpenCV functions
32  * calls to detect potential memory leaks in testKeyPoint-2.cpp.
33  */
34 
42 #include <iostream>
43 
44 #include <visp3/core/vpConfig.h>
45 
46 #if defined(HAVE_OPENCV_IMGPROC) && \
47  ((VISP_HAVE_OPENCV_VERSION < 0x050000) && defined(HAVE_OPENCV_CALIB3D) && defined(HAVE_OPENCV_FEATURES2D)) || \
48  ((VISP_HAVE_OPENCV_VERSION >= 0x050000) && defined(HAVE_OPENCV_3D) && defined(HAVE_OPENCV_FEATURES))
49 
50 #include <visp3/core/vpHomogeneousMatrix.h>
51 #include <visp3/core/vpImage.h>
52 #include <visp3/core/vpIoTools.h>
53 #include <visp3/gui/vpDisplayFactory.h>
54 #include <visp3/io/vpImageIo.h>
55 #include <visp3/io/vpParseArgv.h>
56 #include <visp3/io/vpVideoReader.h>
57 #include <visp3/mbt/vpMbEdgeTracker.h>
58 #include <visp3/vision/vpKeyPoint.h>
59 
60 // List of allowed command line options
61 #define GETOPTARGS "cdh"
62 
63 #ifdef ENABLE_VISP_NAMESPACE
64 using namespace VISP_NAMESPACE_NAME;
65 #endif
66 
67 void usage(const char *name, const char *badparam);
68 bool getOptions(int argc, const char **argv, bool &click_allowed, bool &display);
69 
78 void usage(const char *name, const char *badparam)
79 {
80  fprintf(stdout, "\n\
81 Test keypoints matching.\n\
82 \n\
83 SYNOPSIS\n\
84  %s [-c] [-d] [-h]\n",
85  name);
86 
87  fprintf(stdout, "\n\
88 OPTIONS: \n\
89 \n\
90  -c\n\
91  Disable the mouse click. Useful to automate the \n\
92  execution of this program without human intervention.\n\
93 \n\
94  -d \n\
95  Turn off the display.\n\
96 \n\
97  -h\n\
98  Print the help.\n");
99 
100  if (badparam)
101  fprintf(stdout, "\nERROR: Bad parameter [%s]\n", badparam);
102 }
103 
115 bool getOptions(int argc, const char **argv, bool &click_allowed, bool &display)
116 {
117  const char *optarg_;
118  int c;
119  while ((c = vpParseArgv::parse(argc, argv, GETOPTARGS, &optarg_)) > 1) {
120 
121  switch (c) {
122  case 'c':
123  click_allowed = false;
124  break;
125  case 'd':
126  display = false;
127  break;
128  case 'h':
129  usage(argv[0], nullptr);
130  return false;
131  break;
132 
133  default:
134  usage(argv[0], optarg_);
135  return false;
136  break;
137  }
138  }
139 
140  if ((c == 1) || (c == -1)) {
141  // standalone param or error
142  usage(argv[0], nullptr);
143  std::cerr << "ERROR: " << std::endl;
144  std::cerr << " Bad argument " << optarg_ << std::endl << std::endl;
145  return false;
146  }
147 
148  return true;
149 }
150 
151 template <typename Type>
152 void run_test(const std::string &env_ipath, bool opt_click_allowed, bool opt_display, vpImage<Type> &I,
153  vpImage<Type> &Imatch, vpImage<Type> &Iref)
154 {
155 #if VISP_HAVE_DATASET_VERSION >= 0x030600
156  std::string ext("png");
157 #else
158  std::string ext("pgm");
159 #endif
160  // Set the path location of the image sequence
161  std::string dirname = vpIoTools::createFilePath(env_ipath, "mbt/cube");
162 
163  // Build the name of the image files
164  std::string filenameRef = vpIoTools::createFilePath(dirname, "image0000." + ext);
165  vpImageIo::read(I, filenameRef);
166  Iref = I;
167  std::string filenameCur = vpIoTools::createFilePath(dirname, "image%04d." + ext);
168 
169  vpDisplay *display = nullptr, *display2 = nullptr;
170 
171  if (opt_display) {
172  Imatch.resize(I.getHeight(), 2 * I.getWidth());
173  Imatch.insert(I, vpImagePoint(0, 0));
174 
175 #ifdef VISP_HAVE_DISPLAY
176  display = vpDisplayFactory::allocateDisplay(I, 0, 0, "ORB keypoints matching");
177  display->setDownScalingFactor(vpDisplay::SCALE_AUTO);
178  display2 = vpDisplayFactory::allocateDisplay(Imatch, 0, (int)I.getHeight() / vpDisplay::getDownScalingFactor(I) + 40, "ORB keypoints matching");
179  display2->setDownScalingFactor(vpDisplay::SCALE_AUTO);
180 #else
181  std::cout << "No image viewer is available..." << std::endl;
182 #endif
183  }
184 
185  vpCameraParameters cam;
186  vpMbEdgeTracker tracker;
187  // Load config for tracker
188  std::string tracker_config_file = vpIoTools::createFilePath(env_ipath, "mbt/cube.xml");
189 
190 #if defined(VISP_HAVE_PUGIXML)
191  tracker.loadConfigFile(tracker_config_file);
192  tracker.getCameraParameters(cam);
193 #else
194  // Corresponding parameters manually set to have an example code
195  vpMe me;
196  me.setMaskSize(5);
197  me.setMaskNumber(180);
198  me.setRange(8);
200  me.setThreshold(20);
201  me.setMu1(0.5);
202  me.setMu2(0.5);
203  me.setSampleStep(4);
204  me.setNbTotalSample(250);
205  tracker.setMovingEdge(me);
206  cam.initPersProjWithoutDistortion(547.7367575, 542.0744058, 338.7036994, 234.5083345);
207  tracker.setCameraParameters(cam);
208  tracker.setNearClippingDistance(0.01);
209  tracker.setFarClippingDistance(100.0);
211 #endif
212 
213  tracker.setAngleAppear(vpMath::rad(89));
214  tracker.setAngleDisappear(vpMath::rad(89));
215 
216  // Load CAO model
217  std::string cao_model_file = vpIoTools::createFilePath(env_ipath, "mbt/cube.cao");
218  tracker.loadModel(cao_model_file);
219 
220  // Initialize the pose
221  std::string init_file = vpIoTools::createFilePath(env_ipath, "mbt/cube.init");
222  if (opt_display && opt_click_allowed) {
223  tracker.initClick(I, init_file);
224  }
225  else {
226  vpHomogeneousMatrix cMoi(0.02044769891, 0.1101505452, 0.5078963719, 2.063603907, 1.110231561, -0.4392789872);
227  tracker.initFromPose(I, cMoi);
228  }
229 
230  // Get the init pose
232  tracker.getPose(cMo);
233 
234  // Init keypoints
235  cv::Ptr<cv::FeatureDetector> detector;
236  cv::Ptr<cv::DescriptorExtractor> extractor;
237  cv::Ptr<cv::DescriptorMatcher> matcher;
238 
239 #if ((VISP_HAVE_OPENCV_VERSION < 0x050000) && defined(HAVE_OPENCV_FEATURES2D)) || ((VISP_HAVE_OPENCV_VERSION >= 0x050000) && defined(HAVE_OPENCV_FEATURES))
240 #if (VISP_HAVE_OPENCV_VERSION >= 0x030000)
241  detector = cv::ORB::create(500, 1.2f, 1);
242  extractor = cv::ORB::create(500, 1.2f, 1);
243 #elif (VISP_HAVE_OPENCV_VERSION >= 0x020301)
244  detector = cv::FeatureDetector::create("ORB");
245  extractor = cv::DescriptorExtractor::create("ORB");
246 #endif
247 #endif
248  matcher = cv::DescriptorMatcher::create("BruteForce-Hamming");
249 
250  // Detect keypoints on the current image
251  std::vector<cv::KeyPoint> trainKeyPoints;
252  cv::Mat matImg;
253  vpImageConvert::convert(I, matImg);
254  detector->detect(matImg, trainKeyPoints);
255 
256  // Keep only keypoints on the cube
257  std::vector<vpPolygon> polygons;
258  std::vector<std::vector<vpPoint> > roisPt;
259  std::pair<std::vector<vpPolygon>, std::vector<std::vector<vpPoint> > > pair = tracker.getPolygonFaces(false);
260  polygons = pair.first;
261  roisPt = pair.second;
262 
263  // Compute the 3D coordinates
264  std::vector<cv::Point3f> points3f;
265  vpKeyPoint::compute3DForPointsInPolygons(cMo, cam, trainKeyPoints, polygons, roisPt, points3f);
266 
267  // Extract descriptors
268  cv::Mat trainDescriptors;
269  extractor->compute(matImg, trainKeyPoints, trainDescriptors);
270 
271  if (trainKeyPoints.size() != (size_t)trainDescriptors.rows || trainKeyPoints.size() != points3f.size()) {
272  throw(vpException(vpException::fatalError, "Problem with training data size !"));
273  }
274 
275  // Init reader for getting the input image sequence
276  vpVideoReader g;
277  g.setFileName(filenameCur);
278  g.open(I);
279  g.acquire(I);
280 
281  bool opt_click = false;
283  while (g.getFrameIndex() < 30) {
284  g.acquire(I);
285 
286  vpImageConvert::convert(I, matImg);
287  std::vector<cv::KeyPoint> queryKeyPoints;
288  detector->detect(matImg, queryKeyPoints);
289 
290  cv::Mat queryDescriptors;
291  extractor->compute(matImg, queryKeyPoints, queryDescriptors);
292 
293  std::vector<std::vector<cv::DMatch> > knn_matches;
294  std::vector<cv::DMatch> matches;
295  matcher->knnMatch(queryDescriptors, trainDescriptors, knn_matches, 2);
296  for (std::vector<std::vector<cv::DMatch> >::const_iterator it = knn_matches.begin(); it != knn_matches.end();
297  ++it) {
298  if (it->size() > 1) {
299  double ratio = (*it)[0].distance / (*it)[1].distance;
300  if (ratio < 0.85) {
301  matches.push_back((*it)[0]);
302  }
303  }
304  }
305 
306  vpPose estimated_pose;
307  for (std::vector<cv::DMatch>::const_iterator it = matches.begin(); it != matches.end(); ++it) {
308  vpPoint pt(points3f[(size_t)(it->trainIdx)].x, points3f[(size_t)(it->trainIdx)].y,
309  points3f[(size_t)(it->trainIdx)].z);
310 
311  double x = 0.0, y = 0.0;
312  vpPixelMeterConversion::convertPoint(cam, queryKeyPoints[(size_t)(it->queryIdx)].pt.x,
313  queryKeyPoints[(size_t)(it->queryIdx)].pt.y, x, y);
314  pt.set_x(x);
315  pt.set_y(y);
316 
317  estimated_pose.addPoint(pt);
318  }
319 
320  bool is_pose_estimated = false;
321  if (estimated_pose.npt >= 4) {
322  try {
323  unsigned int nb_inliers = static_cast<unsigned int>(0.7 * estimated_pose.npt);
324  estimated_pose.setRansacNbInliersToReachConsensus(nb_inliers);
325  estimated_pose.setRansacThreshold(0.001);
326  estimated_pose.setRansacMaxTrials(500);
327  if (estimated_pose.computePose(vpPose::RANSAC, cMo)) {
328  is_pose_estimated = true; // success
329  }
330  else {
331  is_pose_estimated = false;
332  }
333  }
334  catch (...) {
335  is_pose_estimated = false;
336  }
337  }
338 
339  if (opt_display) {
341 
342  Imatch.insert(I, vpImagePoint(0, Iref.getWidth()));
343  vpDisplay::display(Imatch);
344  for (std::vector<cv::DMatch>::const_iterator it = matches.begin(); it != matches.end(); ++it) {
345  vpImagePoint leftPt(trainKeyPoints[(size_t)it->trainIdx].pt.y, trainKeyPoints[(size_t)it->trainIdx].pt.x);
346  vpImagePoint rightPt(queryKeyPoints[(size_t)it->queryIdx].pt.y,
347  queryKeyPoints[(size_t)it->queryIdx].pt.x + Iref.getWidth());
348  vpDisplay::displayLine(Imatch, leftPt, rightPt, vpColor::green);
349  }
350 
351  if (is_pose_estimated) {
352  tracker.setPose(I, cMo);
353  tracker.display(I, cMo, cam, vpColor::red);
354  vpDisplay::displayFrame(I, cMo, cam, 0.05, vpColor::none);
355  }
356 
357  vpDisplay::flush(Imatch);
358  vpDisplay::flush(I);
359  }
360 
361  // Click requested to process next image
362  if (opt_click_allowed && opt_display) {
363  if (opt_click) {
364  vpDisplay::getClick(I, button, true);
365  if (button == vpMouseButton::button3) {
366  opt_click = false;
367  }
368  }
369  else {
370  // Use right click to enable/disable step by step tracking
371  if (vpDisplay::getClick(I, button, false)) {
372  if (button == vpMouseButton::button3) {
373  opt_click = true;
374  }
375  else if (button == vpMouseButton::button1) {
376  break;
377  }
378  }
379  }
380  }
381  }
382 
383  if (display) {
384  delete display;
385  }
386  if (display2) {
387  delete display2;
388  }
389 }
390 
391 int main(int argc, const char **argv)
392 {
393  try {
394  std::string env_ipath;
395  bool opt_click_allowed = true;
396  bool opt_display = true;
397 
398  // Read the command line options
399  if (getOptions(argc, argv, opt_click_allowed, opt_display) == false) {
400  return EXIT_FAILURE;
401  }
402 
403  // Get the visp-images-data package path or VISP_INPUT_IMAGE_PATH
404  // environment variable value
405  env_ipath = vpIoTools::getViSPImagesDataPath();
406 
407  if (env_ipath.empty()) {
408  std::cerr << "Please set the VISP_INPUT_IMAGE_PATH environment "
409  "variable value."
410  << std::endl;
411  return EXIT_FAILURE;
412  }
413 
414  {
415  vpImage<unsigned char> I, Imatch, Iref;
416 
417  std::cout << "-- Test on gray level images" << std::endl;
418  run_test(env_ipath, opt_click_allowed, opt_display, I, Imatch, Iref);
419  }
420 
421  {
422  vpImage<vpRGBa> I, Imatch, Iref;
423 
424  std::cout << "-- Test on color images" << std::endl;
425  run_test(env_ipath, opt_click_allowed, opt_display, I, Imatch, Iref);
426  }
427 
428  }
429  catch (const vpException &e) {
430  std::cerr << e.what() << std::endl;
431  return EXIT_FAILURE;
432  }
433 
434  std::cout << "testKeyPoint-4 is ok !" << std::endl;
435  return EXIT_SUCCESS;
436 }
437 
438 #else
439 int main()
440 {
441  std::cerr << "You need OpenCV library." << std::endl;
442 
443  return EXIT_SUCCESS;
444 }
445 
446 #endif
Generic class defining intrinsic camera parameters.
void initPersProjWithoutDistortion(double px, double py, double u0, double v0)
static const vpColor red
Definition: vpColor.h:198
static const vpColor none
Definition: vpColor.h:210
static const vpColor green
Definition: vpColor.h:201
Class that defines generic functionalities for display.
Definition: vpDisplay.h:178
static bool getClick(const vpImage< unsigned char > &I, bool blocking=true)
static void display(const vpImage< unsigned char > &I)
static void displayLine(const vpImage< unsigned char > &I, const vpImagePoint &ip1, const vpImagePoint &ip2, const vpColor &color, unsigned int thickness=1, bool segment=true)
static void displayFrame(const vpImage< unsigned char > &I, const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam, double size, const vpColor &color=vpColor::none, unsigned int thickness=1, const vpImagePoint &offset=vpImagePoint(0, 0), const std::string &frameName="", const vpColor &textColor=vpColor::black, const vpImagePoint &textOffset=vpImagePoint(15, 15))
static void flush(const vpImage< unsigned char > &I)
@ SCALE_AUTO
Definition: vpDisplay.h:184
unsigned int getDownScalingFactor()
Definition: vpDisplay.h:221
error that can be emitted by ViSP classes.
Definition: vpException.h:60
@ fatalError
Fatal error.
Definition: vpException.h:72
const char * what() const
Definition: vpException.cpp:71
Implementation of an homogeneous matrix and operations on such kind of matrices.
static void convert(const vpImage< unsigned char > &src, vpImage< vpRGBa > &dest)
static void read(vpImage< unsigned char > &I, const std::string &filename, int backend=IO_DEFAULT_BACKEND)
Definition: vpImageIo.cpp:147
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
Definition: vpImagePoint.h:82
Definition of the vpImage class member functions.
Definition: vpImage.h:131
unsigned int getWidth() const
Definition: vpImage.h:242
void resize(unsigned int h, unsigned int w)
resize the image : Image initialization
Definition: vpImage.h:544
void insert(const vpImage< Type > &src, const vpImagePoint &topLeft)
Definition: vpImage.h:639
unsigned int getHeight() const
Definition: vpImage.h:181
static std::string getViSPImagesDataPath()
Definition: vpIoTools.cpp:1053
static std::string createFilePath(const std::string &parent, const std::string &child)
Definition: vpIoTools.cpp:1427
static void compute3DForPointsInPolygons(const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam, std::vector< cv::KeyPoint > &candidates, const std::vector< vpPolygon > &polygons, const std::vector< std::vector< vpPoint > > &roisPt, std::vector< cv::Point3f > &points, cv::Mat *descriptors=nullptr)
Definition: vpKeyPoint.cpp:481
static double rad(double deg)
Definition: vpMath.h:129
Make the complete tracking of an object by using its CAD model.
virtual void setNearClippingDistance(const double &dist) VP_OVERRIDE
virtual void display(const vpImage< unsigned char > &I, const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam, const vpColor &col, unsigned int thickness=1, bool displayFullModel=false) VP_OVERRIDE
virtual void setFarClippingDistance(const double &dist) VP_OVERRIDE
virtual void setClipping(const unsigned int &flags) VP_OVERRIDE
void setMovingEdge(const vpMe &me)
virtual void setCameraParameters(const vpCameraParameters &cam) VP_OVERRIDE
virtual void setPose(const vpImage< unsigned char > &I, const vpHomogeneousMatrix &cdMo) VP_OVERRIDE
virtual void loadConfigFile(const std::string &configFile, bool verbose=true) VP_OVERRIDE
virtual void getCameraParameters(vpCameraParameters &cam) const
Definition: vpMbTracker.h:250
virtual void getPose(vpHomogeneousMatrix &cMo) const
Definition: vpMbTracker.h:416
virtual void setAngleDisappear(const double &a)
Definition: vpMbTracker.h:483
virtual void initFromPose(const vpImage< unsigned char > &I, const std::string &initFile)
virtual void loadModel(const std::string &modelFile, bool verbose=false, const vpHomogeneousMatrix &T=vpHomogeneousMatrix())
virtual void setAngleAppear(const double &a)
Definition: vpMbTracker.h:472
virtual std::pair< std::vector< vpPolygon >, std::vector< std::vector< vpPoint > > > getPolygonFaces(bool orderPolygons=true, bool useVisibility=true, bool clipPolygon=false)
virtual unsigned int getClipping() const
Definition: vpMbTracker.h:258
Definition: vpMe.h:134
void setMu1(const double &mu_1)
Definition: vpMe.h:385
void setRange(const unsigned int &range)
Definition: vpMe.h:415
void setLikelihoodThresholdType(const vpLikelihoodThresholdType likelihood_threshold_type)
Definition: vpMe.h:505
void setNbTotalSample(const int &ntotal_sample)
Definition: vpMe.h:399
void setMaskNumber(const unsigned int &mask_number)
Definition: vpMe.cpp:552
void setThreshold(const double &threshold)
Definition: vpMe.h:466
void setSampleStep(const double &sample_step)
Definition: vpMe.h:422
void setMaskSize(const unsigned int &mask_size)
Definition: vpMe.cpp:560
void setMu2(const double &mu_2)
Definition: vpMe.h:392
@ NORMALIZED_THRESHOLD
Definition: vpMe.h:145
static bool parse(int *argcPtr, const char **argv, vpArgvInfo *argTable, int flags)
Definition: vpParseArgv.cpp:70
static void convertPoint(const vpCameraParameters &cam, const double &u, const double &v, double &x, double &y)
Class that defines a 3D point in the object frame and allows forward projection of a 3D point in the ...
Definition: vpPoint.h:79
Class used for pose computation from N points (pose from point only). Some of the algorithms implemen...
Definition: vpPose.h:77
void setRansacMaxTrials(const int &rM)
Definition: vpPose.h:406
void addPoint(const vpPoint &P)
Definition: vpPose.cpp:96
void setRansacNbInliersToReachConsensus(const unsigned int &nbC)
Definition: vpPose.h:387
@ RANSAC
Definition: vpPose.h:87
unsigned int npt
Number of point used in pose computation.
Definition: vpPose.h:113
bool computePose(vpPoseMethodType method, vpHomogeneousMatrix &cMo, FuncCheckValidityPose func=nullptr)
Definition: vpPose.cpp:385
void setRansacThreshold(const double &t)
Definition: vpPose.h:392
Class that enables to manipulate easily a video file or a sequence of images. As it inherits from the...
void acquire(vpImage< vpRGBa > &I)
void open(vpImage< vpRGBa > &I)
void setFileName(const std::string &filename)
long getFrameIndex() const
vpDisplay * allocateDisplay()
Return a newly allocated vpDisplay specialization if a GUI library is available or nullptr otherwise.