Visual Servoing Platform  version 3.2.1 under development (2019-07-16)
vpKeyPoint.h
1 /****************************************************************************
2  *
3  * ViSP, open source Visual Servoing Platform software.
4  * Copyright (C) 2005 - 2019 by Inria. All rights reserved.
5  *
6  * This software is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  * See the file LICENSE.txt at the root directory of this source
11  * distribution for additional information about the GNU GPL.
12  *
13  * For using ViSP with software that can not be combined with the GNU
14  * GPL, please contact Inria about acquiring a ViSP Professional
15  * Edition License.
16  *
17  * See http://visp.inria.fr for more information.
18  *
19  * This software was developed at:
20  * Inria Rennes - Bretagne Atlantique
21  * Campus Universitaire de Beaulieu
22  * 35042 Rennes Cedex
23  * France
24  *
25  * If you have questions regarding the use of this file, please contact
26  * Inria at visp@inria.fr
27  *
28  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
29  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
30  *
31  * Description:
32  * Key point functionalities.
33  *
34  * Authors:
35  * Souriya Trinh
36  *
37  *****************************************************************************/
38 #ifndef _vpKeyPoint_h_
39 #define _vpKeyPoint_h_
40 
41 #include <algorithm> // std::transform
42 #include <float.h> // DBL_MAX
43 #include <fstream> // std::ofstream
44 #include <limits>
45 #include <map> // std::map
46 #include <numeric> // std::accumulate
47 #include <stdlib.h> // srand, rand
48 #include <time.h> // time
49 #include <vector> // std::vector
50 
51 #include <visp3/core/vpConfig.h>
52 #include <visp3/core/vpDisplay.h>
53 #include <visp3/core/vpImageConvert.h>
54 #include <visp3/core/vpPixelMeterConversion.h>
55 #include <visp3/core/vpPlane.h>
56 #include <visp3/core/vpPoint.h>
57 #include <visp3/vision/vpBasicKeyPoint.h>
58 #include <visp3/vision/vpPose.h>
59 #ifdef VISP_HAVE_MODULE_IO
60 # include <visp3/io/vpImageIo.h>
61 #endif
62 #include <visp3/core/vpConvert.h>
63 #include <visp3/core/vpCylinder.h>
64 #include <visp3/core/vpMeterPixelConversion.h>
65 #include <visp3/core/vpPolygon.h>
66 #include <visp3/vision/vpXmlConfigParserKeyPoint.h>
67 
68 // Require at least OpenCV >= 2.1.1
69 #if (VISP_HAVE_OPENCV_VERSION >= 0x020101)
70 
71 # include <opencv2/calib3d/calib3d.hpp>
72 # include <opencv2/features2d/features2d.hpp>
73 # include <opencv2/imgproc/imgproc.hpp>
74 
75 # if (VISP_HAVE_OPENCV_VERSION >= 0x040000) // Require opencv >= 4.0.0
76 # include <opencv2/imgproc/imgproc_c.h>
77 # include <opencv2/imgproc.hpp>
78 # endif
79 
80 # if defined(VISP_HAVE_OPENCV_XFEATURES2D) // OpenCV >= 3.0.0
81 # include <opencv2/xfeatures2d.hpp>
82 # elif defined(VISP_HAVE_OPENCV_NONFREE) && (VISP_HAVE_OPENCV_VERSION >= 0x020400) && \
83  (VISP_HAVE_OPENCV_VERSION < 0x030000)
84 # include <opencv2/nonfree/nonfree.hpp>
85 # endif
86 
224 class VISP_EXPORT vpKeyPoint : public vpBasicKeyPoint
225 {
226 
227 public:
230  constantFactorDistanceThreshold,
232  stdDistanceThreshold,
234  ratioDistanceThreshold,
237  stdAndRatioDistanceThreshold,
239  noFilterMatching
240  };
241 
244  detectionThreshold,
246  detectionScore
249  };
250 
252  typedef enum {
256  pgmImageFormat
258 
261 #if (VISP_HAVE_OPENCV_VERSION >= 0x020403)
268 #if (VISP_HAVE_OPENCV_VERSION < 0x030000) || (defined(VISP_HAVE_OPENCV_XFEATURES2D))
269  DETECTOR_STAR,
270 #endif
271 #if defined(VISP_HAVE_OPENCV_NONFREE) || defined(VISP_HAVE_OPENCV_XFEATURES2D)
274 #endif
275 #if (VISP_HAVE_OPENCV_VERSION >= 0x030000)
279 #endif
280 #if (VISP_HAVE_OPENCV_VERSION >= 0x030100) && defined(VISP_HAVE_OPENCV_XFEATURES2D)
281  DETECTOR_MSD,
282 #endif
283 #endif
284  DETECTOR_TYPE_SIZE
285  };
286 
289 #if (VISP_HAVE_OPENCV_VERSION >= 0x020403)
292 #if (VISP_HAVE_OPENCV_VERSION < 0x030000) || (defined(VISP_HAVE_OPENCV_XFEATURES2D))
293  DESCRIPTOR_FREAK,
294  DESCRIPTOR_BRIEF,
295 #endif
296 #if defined(VISP_HAVE_OPENCV_NONFREE) || defined(VISP_HAVE_OPENCV_XFEATURES2D)
299 #endif
300 #if (VISP_HAVE_OPENCV_VERSION >= 0x030000)
303 #if defined(VISP_HAVE_OPENCV_XFEATURES2D)
304  DESCRIPTOR_DAISY,
305  DESCRIPTOR_LATCH,
306 #endif
307 #endif
308 #if (VISP_HAVE_OPENCV_VERSION >= 0x030200) && defined(VISP_HAVE_OPENCV_XFEATURES2D)
309  DESCRIPTOR_VGG,
310  DESCRIPTOR_BoostDesc,
311 #endif
312 #endif
313  DESCRIPTOR_TYPE_SIZE
314  };
315 
316  vpKeyPoint(const vpFeatureDetectorType &detectorType, const vpFeatureDescriptorType &descriptorType,
317  const std::string &matcherName, const vpFilterMatchingType &filterType = ratioDistanceThreshold);
318  vpKeyPoint(const std::string &detectorName = "ORB", const std::string &extractorName = "ORB",
319  const std::string &matcherName = "BruteForce-Hamming",
320  const vpFilterMatchingType &filterType = ratioDistanceThreshold);
321  vpKeyPoint(const std::vector<std::string> &detectorNames, const std::vector<std::string> &extractorNames,
322  const std::string &matcherName = "BruteForce",
323  const vpFilterMatchingType &filterType = ratioDistanceThreshold);
324 
325  unsigned int buildReference(const vpImage<unsigned char> &I);
326  unsigned int buildReference(const vpImage<unsigned char> &I, const vpImagePoint &iP, const unsigned int height,
327  const unsigned int width);
328  unsigned int buildReference(const vpImage<unsigned char> &I, const vpRect &rectangle);
329 
330  void buildReference(const vpImage<unsigned char> &I, std::vector<cv::KeyPoint> &trainKeyPoint,
331  std::vector<cv::Point3f> &points3f, const bool append = false, const int class_id = -1);
332  void buildReference(const vpImage<unsigned char> &I, const std::vector<cv::KeyPoint> &trainKeyPoint,
333  const cv::Mat &trainDescriptors, const std::vector<cv::Point3f> &points3f,
334  const bool append = false, const int class_id = -1);
335 
336  unsigned int buildReference(const vpImage<vpRGBa> &I_color);
337  unsigned int buildReference(const vpImage<vpRGBa> &I_color, const vpImagePoint &iP, const unsigned int height,
338  const unsigned int width);
339  unsigned int buildReference(const vpImage<vpRGBa> &I_color, const vpRect &rectangle);
340 
341  void buildReference(const vpImage<vpRGBa> &I_color, std::vector<cv::KeyPoint> &trainKeyPoint,
342  std::vector<cv::Point3f> &points3f, const bool append = false, const int class_id = -1);
343  void buildReference(const vpImage<vpRGBa> &I, const std::vector<cv::KeyPoint> &trainKeyPoint,
344  const cv::Mat &trainDescriptors, const std::vector<cv::Point3f> &points3f,
345  const bool append = false, const int class_id = -1);
346 
347  static void compute3D(const cv::KeyPoint &candidate, const std::vector<vpPoint> &roi, const vpCameraParameters &cam,
348  const vpHomogeneousMatrix &cMo, cv::Point3f &point);
349 
350  static void compute3D(const vpImagePoint &candidate, const std::vector<vpPoint> &roi, const vpCameraParameters &cam,
351  const vpHomogeneousMatrix &cMo, vpPoint &point);
352 
353  static void compute3DForPointsInPolygons(const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam,
354  std::vector<cv::KeyPoint> &candidates,
355  const std::vector<vpPolygon> &polygons,
356  const std::vector<std::vector<vpPoint> > &roisPt,
357  std::vector<cv::Point3f> &points, cv::Mat *descriptors = NULL);
358 
359  static void compute3DForPointsInPolygons(const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam,
360  std::vector<vpImagePoint> &candidates,
361  const std::vector<vpPolygon> &polygons,
362  const std::vector<std::vector<vpPoint> > &roisPt,
363  std::vector<vpPoint> &points, cv::Mat *descriptors = NULL);
364 
365  static void
366  compute3DForPointsOnCylinders(const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam,
367  std::vector<cv::KeyPoint> &candidates, const std::vector<vpCylinder> &cylinders,
368  const std::vector<std::vector<std::vector<vpImagePoint> > > &vectorOfCylinderRois,
369  std::vector<cv::Point3f> &points, cv::Mat *descriptors = NULL);
370 
371  static void
372  compute3DForPointsOnCylinders(const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam,
373  std::vector<vpImagePoint> &candidates, const std::vector<vpCylinder> &cylinders,
374  const std::vector<std::vector<std::vector<vpImagePoint> > > &vectorOfCylinderRois,
375  std::vector<vpPoint> &points, cv::Mat *descriptors = NULL);
376 
377  bool computePose(const std::vector<cv::Point2f> &imagePoints, const std::vector<cv::Point3f> &objectPoints,
378  const vpCameraParameters &cam, vpHomogeneousMatrix &cMo, std::vector<int> &inlierIndex,
379  double &elapsedTime, bool (*func)(const vpHomogeneousMatrix &) = NULL);
380 
381  bool computePose(const std::vector<vpPoint> &objectVpPoints, vpHomogeneousMatrix &cMo, std::vector<vpPoint> &inliers,
382  double &elapsedTime, bool (*func)(const vpHomogeneousMatrix &) = NULL);
383 
384  bool computePose(const std::vector<vpPoint> &objectVpPoints, vpHomogeneousMatrix &cMo, std::vector<vpPoint> &inliers,
385  std::vector<unsigned int> &inlierIndex, double &elapsedTime,
386  bool (*func)(const vpHomogeneousMatrix &) = NULL);
387 
388  void createImageMatching(vpImage<unsigned char> &IRef, vpImage<unsigned char> &ICurrent,
389  vpImage<unsigned char> &IMatching);
390  void createImageMatching(vpImage<unsigned char> &ICurrent, vpImage<unsigned char> &IMatching);
391 
392  void createImageMatching(vpImage<unsigned char> &IRef, vpImage<vpRGBa> &ICurrent,
393  vpImage<vpRGBa> &IMatching);
394  void createImageMatching(vpImage<vpRGBa> &ICurrent, vpImage<vpRGBa> &IMatching);
395 
396  void detect(const vpImage<unsigned char> &I, std::vector<cv::KeyPoint> &keyPoints,
397  const vpRect &rectangle = vpRect());
398  void detect(const vpImage<vpRGBa> &I_color, std::vector<cv::KeyPoint> &keyPoints,
399  const vpRect &rectangle = vpRect());
400  void detect(const cv::Mat &matImg, std::vector<cv::KeyPoint> &keyPoints, const cv::Mat &mask = cv::Mat());
401  void detect(const vpImage<unsigned char> &I, std::vector<cv::KeyPoint> &keyPoints, double &elapsedTime,
402  const vpRect &rectangle = vpRect());
403  void detect(const vpImage<vpRGBa> &I_color, std::vector<cv::KeyPoint> &keyPoints, double &elapsedTime,
404  const vpRect &rectangle = vpRect());
405  void detect(const cv::Mat &matImg, std::vector<cv::KeyPoint> &keyPoints, double &elapsedTime,
406  const cv::Mat &mask = cv::Mat());
407 
408  void detectExtractAffine(const vpImage<unsigned char> &I, std::vector<std::vector<cv::KeyPoint> > &listOfKeypoints,
409  std::vector<cv::Mat> &listOfDescriptors,
410  std::vector<vpImage<unsigned char> > *listOfAffineI = NULL);
411 
412  void display(const vpImage<unsigned char> &IRef, const vpImage<unsigned char> &ICurrent, unsigned int size = 3);
413  void display(const vpImage<unsigned char> &ICurrent, unsigned int size = 3, const vpColor &color = vpColor::green);
414  void display(const vpImage<vpRGBa> &IRef, const vpImage<vpRGBa> &ICurrent, unsigned int size = 3);
415  void display(const vpImage<vpRGBa> &ICurrent, unsigned int size = 3, const vpColor &color = vpColor::green);
416 
417  void displayMatching(const vpImage<unsigned char> &IRef, vpImage<unsigned char> &IMatching, unsigned int crossSize,
418  unsigned int lineThickness = 1, const vpColor &color = vpColor::green);
419  void displayMatching(const vpImage<unsigned char> &ICurrent, vpImage<unsigned char> &IMatching,
420  const std::vector<vpImagePoint> &ransacInliers = std::vector<vpImagePoint>(),
421  unsigned int crossSize = 3, unsigned int lineThickness = 1);
422  void displayMatching(const vpImage<unsigned char> &IRef, vpImage<vpRGBa> &IMatching, unsigned int crossSize,
423  unsigned int lineThickness = 1, const vpColor &color = vpColor::green);
424  void displayMatching(const vpImage<vpRGBa> &IRef, vpImage<vpRGBa> &IMatching, unsigned int crossSize,
425  unsigned int lineThickness = 1, const vpColor &color = vpColor::green);
426  void displayMatching(const vpImage<vpRGBa> &ICurrent, vpImage<vpRGBa> &IMatching,
427  const std::vector<vpImagePoint> &ransacInliers = std::vector<vpImagePoint>(),
428  unsigned int crossSize = 3, unsigned int lineThickness = 1);
429 
430  void extract(const vpImage<unsigned char> &I, std::vector<cv::KeyPoint> &keyPoints, cv::Mat &descriptors,
431  std::vector<cv::Point3f> *trainPoints = NULL);
432  void extract(const vpImage<vpRGBa> &I_color, std::vector<cv::KeyPoint> &keyPoints, cv::Mat &descriptors,
433  std::vector<cv::Point3f> *trainPoints = NULL);
434  void extract(const cv::Mat &matImg, std::vector<cv::KeyPoint> &keyPoints, cv::Mat &descriptors,
435  std::vector<cv::Point3f> *trainPoints = NULL);
436  void extract(const vpImage<unsigned char> &I, std::vector<cv::KeyPoint> &keyPoints, cv::Mat &descriptors,
437  double &elapsedTime, std::vector<cv::Point3f> *trainPoints = NULL);
438  void extract(const vpImage<vpRGBa> &I_color, std::vector<cv::KeyPoint> &keyPoints, cv::Mat &descriptors,
439  double &elapsedTime, std::vector<cv::Point3f> *trainPoints = NULL);
440  void extract(const cv::Mat &matImg, std::vector<cv::KeyPoint> &keyPoints, cv::Mat &descriptors, double &elapsedTime,
441  std::vector<cv::Point3f> *trainPoints = NULL);
442 
453  {
454  if (!m_computeCovariance) {
455  std::cout << "Warning : The covariance matrix has not been computed. "
456  "See setCovarianceComputation() to do it."
457  << std::endl;
458  return vpMatrix();
459  }
460 
461  if (m_computeCovariance && !m_useRansacVVS) {
462  std::cout << "Warning : The covariance matrix can only be computed "
463  "with a Virtual Visual Servoing approach."
464  << std::endl
465  << "Use setUseRansacVVS(true) to choose to use a pose "
466  "estimation method based on a Virtual Visual Servoing "
467  "approach."
468  << std::endl;
469  return vpMatrix();
470  }
471 
472  return m_covarianceMatrix;
473  }
474 
480  inline double getDetectionTime() const { return m_detectionTime; }
481 
489  inline cv::Ptr<cv::FeatureDetector> getDetector(const vpFeatureDetectorType &type) const
490  {
491  std::map<vpFeatureDetectorType, std::string>::const_iterator it_name = m_mapOfDetectorNames.find(type);
492  if (it_name == m_mapOfDetectorNames.end()) {
493  std::cerr << "Internal problem with the feature type and the "
494  "corresponding name!"
495  << std::endl;
496  }
497 
498  std::map<std::string, cv::Ptr<cv::FeatureDetector> >::const_iterator findDetector =
499  m_detectors.find(it_name->second);
500  if (findDetector != m_detectors.end()) {
501  return findDetector->second;
502  }
503 
504  std::cerr << "Cannot find: " << it_name->second << std::endl;
505  return cv::Ptr<cv::FeatureDetector>();
506  }
507 
515  inline cv::Ptr<cv::FeatureDetector> getDetector(const std::string &name) const
516  {
517  std::map<std::string, cv::Ptr<cv::FeatureDetector> >::const_iterator findDetector = m_detectors.find(name);
518  if (findDetector != m_detectors.end()) {
519  return findDetector->second;
520  }
521 
522  std::cerr << "Cannot find: " << name << std::endl;
523  return cv::Ptr<cv::FeatureDetector>();
524  }
525 
529  inline std::map<vpFeatureDetectorType, std::string> getDetectorNames() const { return m_mapOfDetectorNames; }
530 
536  inline double getExtractionTime() const { return m_extractionTime; }
537 
545  inline cv::Ptr<cv::DescriptorExtractor> getExtractor(const vpFeatureDescriptorType &type) const
546  {
547  std::map<vpFeatureDescriptorType, std::string>::const_iterator it_name = m_mapOfDescriptorNames.find(type);
548  if (it_name == m_mapOfDescriptorNames.end()) {
549  std::cerr << "Internal problem with the feature type and the "
550  "corresponding name!"
551  << std::endl;
552  }
553 
554  std::map<std::string, cv::Ptr<cv::DescriptorExtractor> >::const_iterator findExtractor =
555  m_extractors.find(it_name->second);
556  if (findExtractor != m_extractors.end()) {
557  return findExtractor->second;
558  }
559 
560  std::cerr << "Cannot find: " << it_name->second << std::endl;
561  return cv::Ptr<cv::DescriptorExtractor>();
562  }
563 
571  inline cv::Ptr<cv::DescriptorExtractor> getExtractor(const std::string &name) const
572  {
573  std::map<std::string, cv::Ptr<cv::DescriptorExtractor> >::const_iterator findExtractor = m_extractors.find(name);
574  if (findExtractor != m_extractors.end()) {
575  return findExtractor->second;
576  }
577 
578  std::cerr << "Cannot find: " << name << std::endl;
579  return cv::Ptr<cv::DescriptorExtractor>();
580  }
581 
585  inline std::map<vpFeatureDescriptorType, std::string> getExtractorNames() const { return m_mapOfDescriptorNames; }
586 
592  inline vpImageFormatType getImageFormat() const { return m_imageFormat; }
593 
599  inline double getMatchingTime() const { return m_matchingTime; }
600 
606  inline cv::Ptr<cv::DescriptorMatcher> getMatcher() const { return m_matcher; }
607 
614  inline std::vector<cv::DMatch> getMatches() const { return m_filteredMatches; }
615 
623  inline std::vector<std::pair<cv::KeyPoint, cv::KeyPoint> > getMatchQueryToTrainKeyPoints() const
624  {
625  std::vector<std::pair<cv::KeyPoint, cv::KeyPoint> > matchQueryToTrainKeyPoints(m_filteredMatches.size());
626  for (size_t i = 0; i < m_filteredMatches.size(); i++) {
627  matchQueryToTrainKeyPoints.push_back(
628  std::pair<cv::KeyPoint, cv::KeyPoint>(m_queryFilteredKeyPoints[(size_t)m_filteredMatches[i].queryIdx],
629  m_trainKeyPoints[(size_t)m_filteredMatches[i].trainIdx]));
630  }
631  return matchQueryToTrainKeyPoints;
632  }
633 
639  inline unsigned int getNbImages() const {
640  return static_cast<unsigned int>(m_mapOfImages.size());
641  }
642 
643  void getObjectPoints(std::vector<cv::Point3f> &objectPoints) const;
644  void getObjectPoints(std::vector<vpPoint> &objectPoints) const;
645 
651  inline double getPoseTime() const { return m_poseTime; }
652 
659  inline cv::Mat getQueryDescriptors() const { return m_queryDescriptors; }
660 
661  void getQueryKeyPoints(std::vector<cv::KeyPoint> &keyPoints) const;
662  void getQueryKeyPoints(std::vector<vpImagePoint> &keyPoints) const;
663 
669  inline std::vector<vpImagePoint> getRansacInliers() const { return m_ransacInliers; }
670 
676  inline std::vector<vpImagePoint> getRansacOutliers() const { return m_ransacOutliers; }
677 
684  inline cv::Mat getTrainDescriptors() const { return m_trainDescriptors; }
685 
686  void getTrainKeyPoints(std::vector<cv::KeyPoint> &keyPoints) const;
687  void getTrainKeyPoints(std::vector<vpImagePoint> &keyPoints) const;
688 
689  void getTrainPoints(std::vector<cv::Point3f> &points) const;
690  void getTrainPoints(std::vector<vpPoint> &points) const;
691 
692  void initMatcher(const std::string &matcherName);
693 
694  void insertImageMatching(const vpImage<unsigned char> &IRef, const vpImage<unsigned char> &ICurrent,
695  vpImage<unsigned char> &IMatching);
696  void insertImageMatching(const vpImage<unsigned char> &ICurrent, vpImage<unsigned char> &IMatching);
697 
698  void insertImageMatching(const vpImage<vpRGBa> &IRef, const vpImage<vpRGBa> &ICurrent,
699  vpImage<vpRGBa> &IMatching);
700  void insertImageMatching(const vpImage<vpRGBa> &ICurrent, vpImage<vpRGBa> &IMatching);
701 
702 #ifdef VISP_HAVE_PUGIXML
703  void loadConfigFile(const std::string &configFile);
704 #endif
705 
706  void loadLearningData(const std::string &filename, const bool binaryMode = false, const bool append = false);
707 
708  void match(const cv::Mat &trainDescriptors, const cv::Mat &queryDescriptors, std::vector<cv::DMatch> &matches,
709  double &elapsedTime);
710 
711  unsigned int matchPoint(const vpImage<unsigned char> &I);
712  unsigned int matchPoint(const vpImage<unsigned char> &I, const vpImagePoint &iP, const unsigned int height,
713  const unsigned int width);
714  unsigned int matchPoint(const vpImage<unsigned char> &I, const vpRect &rectangle);
715 
717  bool (*func)(const vpHomogeneousMatrix &) = NULL, const vpRect &rectangle = vpRect());
719  double &error, double &elapsedTime, bool (*func)(const vpHomogeneousMatrix &) = NULL,
720  const vpRect &rectangle = vpRect());
721 
722  bool matchPointAndDetect(const vpImage<unsigned char> &I, vpRect &boundingBox, vpImagePoint &centerOfGravity,
723  const bool isPlanarObject = true, std::vector<vpImagePoint> *imPts1 = NULL,
724  std::vector<vpImagePoint> *imPts2 = NULL, double *meanDescriptorDistance = NULL,
725  double *detectionScore = NULL, const vpRect &rectangle = vpRect());
726 
727  bool matchPointAndDetect(const vpImage<unsigned char> &I, const vpCameraParameters &cam, vpHomogeneousMatrix &cMo,
728  double &error, double &elapsedTime, vpRect &boundingBox, vpImagePoint &centerOfGravity,
729  bool (*func)(const vpHomogeneousMatrix &) = NULL, const vpRect &rectangle = vpRect());
730 
731  unsigned int matchPoint(const vpImage<vpRGBa> &I_color);
732  unsigned int matchPoint(const vpImage<vpRGBa> &I_color, const vpImagePoint &iP, const unsigned int height,
733  const unsigned int width);
734  unsigned int matchPoint(const vpImage<vpRGBa> &I_color, const vpRect &rectangle);
735 
736  bool matchPoint(const vpImage<vpRGBa> &I_color, const vpCameraParameters &cam, vpHomogeneousMatrix &cMo,
737  bool (*func)(const vpHomogeneousMatrix &) = NULL, const vpRect &rectangle = vpRect());
738  bool matchPoint(const vpImage<vpRGBa> &I_color, const vpCameraParameters &cam, vpHomogeneousMatrix &cMo,
739  double &error, double &elapsedTime, bool (*func)(const vpHomogeneousMatrix &) = NULL,
740  const vpRect &rectangle = vpRect());
741 
742  void reset();
743 
744  void saveLearningData(const std::string &filename, const bool binaryMode = false,
745  const bool saveTrainingImages = true);
746 
753  inline void setCovarianceComputation(const bool &flag)
754  {
755  m_computeCovariance = flag;
756  if (!m_useRansacVVS) {
757  std::cout << "Warning : The covariance matrix can only be computed "
758  "with a Virtual Visual Servoing approach."
759  << std::endl
760  << "Use setUseRansacVVS(true) to choose to use a pose "
761  "estimation method based on a Virtual "
762  "Visual Servoing approach."
763  << std::endl;
764  }
765  }
766 
772  inline void setDetectionMethod(const vpDetectionMethodType &method) { m_detectionMethod = method; }
773 
779  inline void setDetector(const vpFeatureDetectorType &detectorType)
780  {
781  m_detectorNames.clear();
782  m_detectorNames.push_back(m_mapOfDetectorNames[detectorType]);
783  m_detectors.clear();
784  initDetector(m_mapOfDetectorNames[detectorType]);
785  }
786 
792  inline void setDetector(const std::string &detectorName)
793  {
794  m_detectorNames.clear();
795  m_detectorNames.push_back(detectorName);
796  m_detectors.clear();
797  initDetector(detectorName);
798  }
799 
800 #if (VISP_HAVE_OPENCV_VERSION >= 0x020400 && VISP_HAVE_OPENCV_VERSION < 0x030000)
801 
809  template <typename T1, typename T2, typename T3>
810  inline void setDetectorParameter(const T1 detectorName, const T2 parameterName, const T3 value)
811  {
812  if (m_detectors.find(detectorName) != m_detectors.end()) {
813  m_detectors[detectorName]->set(parameterName, value);
814  }
815  }
816 #endif
817 
824  inline void setDetectors(const std::vector<std::string> &detectorNames)
825  {
826  m_detectorNames.clear();
827  m_detectors.clear();
828  m_detectorNames = detectorNames;
829  initDetectors(m_detectorNames);
830  }
831 
837  inline void setExtractor(const vpFeatureDescriptorType &extractorType)
838  {
839  m_extractorNames.clear();
840  m_extractorNames.push_back(m_mapOfDescriptorNames[extractorType]);
841  m_extractors.clear();
842  initExtractor(m_mapOfDescriptorNames[extractorType]);
843  }
844 
851  inline void setExtractor(const std::string &extractorName)
852  {
853  m_extractorNames.clear();
854  m_extractorNames.push_back(extractorName);
855  m_extractors.clear();
856  initExtractor(extractorName);
857  }
858 
859 #if (VISP_HAVE_OPENCV_VERSION >= 0x020400 && VISP_HAVE_OPENCV_VERSION < 0x030000)
860 
868  template <typename T1, typename T2, typename T3>
869  inline void setExtractorParameter(const T1 extractorName, const T2 parameterName, const T3 value)
870  {
871  if (m_extractors.find(extractorName) != m_extractors.end()) {
872  m_extractors[extractorName]->set(parameterName, value);
873  }
874  }
875 #endif
876 
883  inline void setExtractors(const std::vector<std::string> &extractorNames)
884  {
885  m_extractorNames.clear();
886  m_extractorNames = extractorNames;
887  m_extractors.clear();
888  initExtractors(m_extractorNames);
889  }
890 
896  inline void setImageFormat(const vpImageFormatType &imageFormat) { m_imageFormat = imageFormat; }
897 
913  inline void setMatcher(const std::string &matcherName)
914  {
915  m_matcherName = matcherName;
916  initMatcher(m_matcherName);
917  }
918 
934  inline void setFilterMatchingType(const vpFilterMatchingType &filterType)
935  {
936  m_filterType = filterType;
937 
938  // Use k-nearest neighbors (knn) to retrieve the two best matches for a
939  // keypoint So this is useful only for ratioDistanceThreshold method
940  if (filterType == ratioDistanceThreshold || filterType == stdAndRatioDistanceThreshold) {
941  m_useKnn = true;
942 
943 #if (VISP_HAVE_OPENCV_VERSION >= 0x020400 && VISP_HAVE_OPENCV_VERSION < 0x030000)
944  if (m_matcher != NULL && m_matcherName == "BruteForce") {
945  // if a matcher is already initialized, disable the crossCheck
946  // because it will not work with knnMatch
947  m_matcher->set("crossCheck", false);
948  }
949 #endif
950  } else {
951  m_useKnn = false;
952 
953 #if (VISP_HAVE_OPENCV_VERSION >= 0x020400 && VISP_HAVE_OPENCV_VERSION < 0x030000)
954  if (m_matcher != NULL && m_matcherName == "BruteForce") {
955  // if a matcher is already initialized, set the crossCheck mode if
956  // necessary
957  m_matcher->set("crossCheck", m_useBruteForceCrossCheck);
958  }
959 #endif
960  }
961  }
962 
969  inline void setMatchingFactorThreshold(const double factor)
970  {
971  if (factor > 0.0) {
972  m_matchingFactorThreshold = factor;
973  } else {
974  throw vpException(vpException::badValue, "The factor must be positive.");
975  }
976  }
977 
983  inline void setMatchingRatioThreshold(const double ratio)
984  {
985  if (ratio > 0.0 && (ratio < 1.0 || std::fabs(ratio - 1.0) < std::numeric_limits<double>::epsilon())) {
986  m_matchingRatioThreshold = ratio;
987  } else {
988  throw vpException(vpException::badValue, "The ratio must be in the interval ]0 ; 1].");
989  }
990  }
991 
998  inline void setRansacConsensusPercentage(const double percentage)
999  {
1000  if (percentage > 0.0 &&
1001  (percentage < 100.0 || std::fabs(percentage - 100.0) < std::numeric_limits<double>::epsilon())) {
1002  m_ransacConsensusPercentage = percentage;
1003  } else {
1004  throw vpException(vpException::badValue, "The percentage must be in the interval ]0 ; 100].");
1005  }
1006  }
1007 
1012  {
1013  m_ransacFilterFlag = flag;
1014  }
1015 
1022  inline void setRansacIteration(const int nbIter)
1023  {
1024  if (nbIter > 0) {
1025  m_nbRansacIterations = nbIter;
1026  } else {
1027  throw vpException(vpException::badValue, "The number of iterations must be greater than zero.");
1028  }
1029  }
1030 
1036  inline void setRansacParallel(const bool parallel)
1037  {
1038  m_ransacParallel = parallel;
1039  }
1040 
1047  inline void setRansacParallelNbThreads(const unsigned int nthreads)
1048  {
1049  m_ransacParallelNbThreads = nthreads;
1050  }
1051 
1059  inline void setRansacReprojectionError(const double reprojectionError)
1060  {
1061  if (reprojectionError > 0.0) {
1062  m_ransacReprojectionError = reprojectionError;
1063  } else {
1064  throw vpException(vpException::badValue, "The Ransac reprojection "
1065  "threshold must be positive "
1066  "as we deal with distance.");
1067  }
1068  }
1069 
1075  inline void setRansacMinInlierCount(const int minCount)
1076  {
1077  if (minCount > 0) {
1078  m_nbRansacMinInlierCount = minCount;
1079  } else {
1080  throw vpException(vpException::badValue, "The minimum number of inliers must be greater than zero.");
1081  }
1082  }
1083 
1090  inline void setRansacThreshold(const double threshold)
1091  {
1092  if (threshold > 0.0) {
1093  m_ransacThreshold = threshold;
1094  } else {
1095  throw vpException(vpException::badValue, "The Ransac threshold must be positive as we deal with distance.");
1096  }
1097  }
1098 
1106  inline void setUseAffineDetection(const bool useAffine) { m_useAffineDetection = useAffine; }
1107 
1108 #if (VISP_HAVE_OPENCV_VERSION >= 0x020400 && VISP_HAVE_OPENCV_VERSION < 0x030000)
1109 
1115  inline void setUseBruteForceCrossCheck(const bool useCrossCheck)
1116  {
1117  // Only available with BruteForce and with k=1 (i.e not used with a
1118  // ratioDistanceThreshold method)
1119  if (m_matcher != NULL && !m_useKnn && m_matcherName == "BruteForce") {
1120  m_matcher->set("crossCheck", useCrossCheck);
1121  } else if (m_matcher != NULL && m_useKnn && m_matcherName == "BruteForce") {
1122  std::cout << "Warning, you try to set the crossCheck parameter with a "
1123  "BruteForce matcher but knn is enabled";
1124  std::cout << " (the filtering method uses a ratio constraint)" << std::endl;
1125  }
1126  }
1127 #endif
1128 
1135  inline void setUseMatchTrainToQuery(const bool useMatchTrainToQuery)
1136  {
1137  m_useMatchTrainToQuery = useMatchTrainToQuery;
1138  }
1139 
1147  inline void setUseRansacConsensusPercentage(const bool usePercentage) { m_useConsensusPercentage = usePercentage; }
1148 
1156  inline void setUseRansacVVS(const bool ransacVVS) { m_useRansacVVS = ransacVVS; }
1157 
1164  inline void setUseSingleMatchFilter(const bool singleMatchFilter) { m_useSingleMatchFilter = singleMatchFilter; }
1165 
1166 private:
1169  bool m_computeCovariance;
1171  vpMatrix m_covarianceMatrix;
1173  int m_currentImageId;
1176  vpDetectionMethodType m_detectionMethod;
1178  double m_detectionScore;
1181  double m_detectionThreshold;
1183  double m_detectionTime;
1185  std::vector<std::string> m_detectorNames;
1188  // with a key based upon the detector name.
1189  std::map<std::string, cv::Ptr<cv::FeatureDetector> > m_detectors;
1191  double m_extractionTime;
1193  std::vector<std::string> m_extractorNames;
1196  // with a key based upon the extractor name.
1197  std::map<std::string, cv::Ptr<cv::DescriptorExtractor> > m_extractors;
1199  std::vector<cv::DMatch> m_filteredMatches;
1201  vpFilterMatchingType m_filterType;
1203  vpImageFormatType m_imageFormat;
1206  std::vector<std::vector<cv::DMatch> > m_knnMatches;
1208  std::map<vpFeatureDescriptorType, std::string> m_mapOfDescriptorNames;
1210  std::map<vpFeatureDetectorType, std::string> m_mapOfDetectorNames;
1213  std::map<int, int> m_mapOfImageId;
1216  std::map<int, vpImage<unsigned char> > m_mapOfImages;
1219  cv::Ptr<cv::DescriptorMatcher> m_matcher;
1221  std::string m_matcherName;
1223  std::vector<cv::DMatch> m_matches;
1225  double m_matchingFactorThreshold;
1227  double m_matchingRatioThreshold;
1229  double m_matchingTime;
1231  std::vector<std::pair<cv::KeyPoint, cv::Point3f> > m_matchRansacKeyPointsToPoints;
1233  int m_nbRansacIterations;
1235  int m_nbRansacMinInlierCount;
1238  std::vector<cv::Point3f> m_objectFilteredPoints;
1240  double m_poseTime;
1243  cv::Mat m_queryDescriptors;
1245  std::vector<cv::KeyPoint> m_queryFilteredKeyPoints;
1247  std::vector<cv::KeyPoint> m_queryKeyPoints;
1250  double m_ransacConsensusPercentage;
1252  vpPose::RANSAC_FILTER_FLAGS m_ransacFilterFlag;
1254  std::vector<vpImagePoint> m_ransacInliers;
1256  std::vector<vpImagePoint> m_ransacOutliers;
1258  bool m_ransacParallel;
1260  unsigned int m_ransacParallelNbThreads;
1263  double m_ransacReprojectionError;
1266  double m_ransacThreshold;
1269  // detected in the train images).
1270  cv::Mat m_trainDescriptors;
1272  std::vector<cv::KeyPoint> m_trainKeyPoints;
1275  std::vector<cv::Point3f> m_trainPoints;
1278  std::vector<vpPoint> m_trainVpPoints;
1281  bool m_useAffineDetection;
1282 #if (VISP_HAVE_OPENCV_VERSION >= 0x020400 && VISP_HAVE_OPENCV_VERSION < 0x030000)
1283  bool m_useBruteForceCrossCheck;
1287 #endif
1288  bool m_useConsensusPercentage;
1292  bool m_useKnn;
1297  bool m_useMatchTrainToQuery;
1299  bool m_useRansacVVS;
1302  bool m_useSingleMatchFilter;
1305 
1306  void affineSkew(double tilt, double phi, cv::Mat &img, cv::Mat &mask, cv::Mat &Ai);
1307 
1308  double computePoseEstimationError(const std::vector<std::pair<cv::KeyPoint, cv::Point3f> > &matchKeyPoints,
1309  const vpCameraParameters &cam, const vpHomogeneousMatrix &cMo_est);
1310 
1311  void filterMatches();
1312 
1313  void init();
1314  void initDetector(const std::string &detectorNames);
1315  void initDetectors(const std::vector<std::string> &detectorNames);
1316 
1317  void initExtractor(const std::string &extractorName);
1318  void initExtractors(const std::vector<std::string> &extractorNames);
1319 
1320  void initFeatureNames();
1321 
1322  inline size_t myKeypointHash(const cv::KeyPoint &kp)
1323  {
1324  size_t _Val = 2166136261U, scale = 16777619U;
1325  Cv32suf u;
1326  u.f = kp.pt.x;
1327  _Val = (scale * _Val) ^ u.u;
1328  u.f = kp.pt.y;
1329  _Val = (scale * _Val) ^ u.u;
1330  u.f = kp.size;
1331  _Val = (scale * _Val) ^ u.u;
1332  // As the keypoint angle can be computed for certain type of keypoint only
1333  // when extracting the corresponding descriptor, the angle field is not
1334  // taking into account for the hash
1335  // u.f = kp.angle; _Val = (scale * _Val) ^ u.u;
1336  u.f = kp.response;
1337  _Val = (scale * _Val) ^ u.u;
1338  _Val = (scale * _Val) ^ ((size_t)kp.octave);
1339  _Val = (scale * _Val) ^ ((size_t)kp.class_id);
1340  return _Val;
1341  }
1342 
1343 #if (VISP_HAVE_OPENCV_VERSION >= 0x030000)
1344  /*
1345  * Adapts a detector to detect points over multiple levels of a Gaussian
1346  * pyramid. Useful for detectors that are not inherently scaled.
1347  * From OpenCV 2.4.11 source code.
1348  */
1349  class PyramidAdaptedFeatureDetector : public cv::FeatureDetector
1350  {
1351  public:
1352  // maxLevel - The 0-based index of the last pyramid layer
1353  PyramidAdaptedFeatureDetector(const cv::Ptr<cv::FeatureDetector> &detector, int maxLevel = 2);
1354 
1355  // TODO implement read/write
1356  virtual bool empty() const;
1357 
1358  protected:
1359  virtual void detect(cv::InputArray image, CV_OUT std::vector<cv::KeyPoint> &keypoints,
1360  cv::InputArray mask = cv::noArray());
1361  virtual void detectImpl(const cv::Mat &image, std::vector<cv::KeyPoint> &keypoints,
1362  const cv::Mat &mask = cv::Mat()) const;
1363 
1364  cv::Ptr<cv::FeatureDetector> detector;
1365  int maxLevel;
1366  };
1367 
1368  /*
1369  * A class filters a vector of keypoints.
1370  * Because now it is difficult to provide a convenient interface for all
1371  * usage scenarios of the keypoints filter class, it has only several needed
1372  * by now static methods.
1373  */
1374  class KeyPointsFilter
1375  {
1376  public:
1377  KeyPointsFilter() {}
1378 
1379  /*
1380  * Remove keypoints within borderPixels of an image edge.
1381  */
1382  static void runByImageBorder(std::vector<cv::KeyPoint> &keypoints, cv::Size imageSize, int borderSize);
1383  /*
1384  * Remove keypoints of sizes out of range.
1385  */
1386  static void runByKeypointSize(std::vector<cv::KeyPoint> &keypoints, float minSize, float maxSize = FLT_MAX);
1387  /*
1388  * Remove keypoints from some image by mask for pixels of this image.
1389  */
1390  static void runByPixelsMask(std::vector<cv::KeyPoint> &keypoints, const cv::Mat &mask);
1391  /*
1392  * Remove duplicated keypoints.
1393  */
1394  static void removeDuplicated(std::vector<cv::KeyPoint> &keypoints);
1395 
1396  /*
1397  * Retain the specified number of the best keypoints (according to the
1398  * response)
1399  */
1400  static void retainBest(std::vector<cv::KeyPoint> &keypoints, int npoints);
1401  };
1402 
1403 #endif
1404 };
1405 
1406 #endif
1407 #endif
Used to indicate that a value is not in the allowed range.
Definition: vpException.h:97
Implementation of a matrix and operations on matrices.
Definition: vpMatrix.h:164
void setRansacIteration(const int nbIter)
Definition: vpKeyPoint.h:1022
class that defines what is a Keypoint. This class provides all the basic elements to implement classe...
void setUseRansacVVS(const bool ransacVVS)
Definition: vpKeyPoint.h:1156
Implementation of an homogeneous matrix and operations on such kind of matrices.
std::vector< std::pair< cv::KeyPoint, cv::KeyPoint > > getMatchQueryToTrainKeyPoints() const
Definition: vpKeyPoint.h:623
void setRansacThreshold(const double threshold)
Definition: vpKeyPoint.h:1090
void setExtractor(const std::string &extractorName)
Definition: vpKeyPoint.h:851
void setUseSingleMatchFilter(const bool singleMatchFilter)
Definition: vpKeyPoint.h:1164
Class to define colors available for display functionnalities.
Definition: vpColor.h:120
std::map< vpFeatureDetectorType, std::string > getDetectorNames() const
Definition: vpKeyPoint.h:529
error that can be emited by ViSP classes.
Definition: vpException.h:71
void setDetectors(const std::vector< std::string > &detectorNames)
Definition: vpKeyPoint.h:824
static const vpColor green
Definition: vpColor.h:183
cv::Ptr< cv::DescriptorExtractor > getExtractor(const vpFeatureDescriptorType &type) const
Definition: vpKeyPoint.h:545
Class that defines what is a point.
Definition: vpPoint.h:58
cv::Mat getQueryDescriptors() const
Definition: vpKeyPoint.h:659
void setExtractors(const std::vector< std::string > &extractorNames)
Definition: vpKeyPoint.h:883
void setMatcher(const std::string &matcherName)
Definition: vpKeyPoint.h:913
void setUseMatchTrainToQuery(const bool useMatchTrainToQuery)
Definition: vpKeyPoint.h:1135
cv::Ptr< cv::DescriptorMatcher > getMatcher() const
Definition: vpKeyPoint.h:606
void setRansacFilterFlag(const vpPose::RANSAC_FILTER_FLAGS &flag)
Definition: vpKeyPoint.h:1011
vpFeatureDetectorType
Definition: vpKeyPoint.h:260
virtual unsigned int buildReference(const vpImage< unsigned char > &I)=0
double getDetectionTime() const
Definition: vpKeyPoint.h:480
void setRansacParallel(const bool parallel)
Definition: vpKeyPoint.h:1036
Generic class defining intrinsic camera parameters.
vpFeatureDescriptorType
Definition: vpKeyPoint.h:288
void setDetector(const vpFeatureDetectorType &detectorType)
Definition: vpKeyPoint.h:779
double getMatchingTime() const
Definition: vpKeyPoint.h:599
void setDetectionMethod(const vpDetectionMethodType &method)
Definition: vpKeyPoint.h:772
unsigned int getNbImages() const
Definition: vpKeyPoint.h:639
cv::Ptr< cv::FeatureDetector > getDetector(const std::string &name) const
Definition: vpKeyPoint.h:515
void setRansacParallelNbThreads(const unsigned int nthreads)
Definition: vpKeyPoint.h:1047
void setRansacMinInlierCount(const int minCount)
Definition: vpKeyPoint.h:1075
vpMatrix getCovarianceMatrix() const
Definition: vpKeyPoint.h:452
void setImageFormat(const vpImageFormatType &imageFormat)
Definition: vpKeyPoint.h:896
virtual void display(const vpImage< unsigned char > &Iref, const vpImage< unsigned char > &Icurrent, unsigned int size=3)=0
void setRansacConsensusPercentage(const double percentage)
Definition: vpKeyPoint.h:998
virtual unsigned int matchPoint(const vpImage< unsigned char > &I)=0
vpImageFormatType getImageFormat() const
Definition: vpKeyPoint.h:592
void setUseAffineDetection(const bool useAffine)
Definition: vpKeyPoint.h:1106
std::map< vpFeatureDescriptorType, std::string > getExtractorNames() const
Definition: vpKeyPoint.h:585
Class that allows keypoints detection (and descriptors extraction) and matching thanks to OpenCV libr...
Definition: vpKeyPoint.h:224
std::vector< vpImagePoint > getRansacOutliers() const
Definition: vpKeyPoint.h:676
vpDetectionMethodType
Definition: vpKeyPoint.h:243
vpFilterMatchingType
Definition: vpKeyPoint.h:229
cv::Mat getTrainDescriptors() const
Definition: vpKeyPoint.h:684
double getExtractionTime() const
Definition: vpKeyPoint.h:536
double getPoseTime() const
Definition: vpKeyPoint.h:651
void setDetector(const std::string &detectorName)
Definition: vpKeyPoint.h:792
Defines a rectangle in the plane.
Definition: vpRect.h:78
cv::Ptr< cv::DescriptorExtractor > getExtractor(const std::string &name) const
Definition: vpKeyPoint.h:571
void setUseRansacConsensusPercentage(const bool usePercentage)
Definition: vpKeyPoint.h:1147
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
Definition: vpImagePoint.h:88
std::vector< vpImagePoint > getRansacInliers() const
Definition: vpKeyPoint.h:669
void setFilterMatchingType(const vpFilterMatchingType &filterType)
Definition: vpKeyPoint.h:934
void setMatchingFactorThreshold(const double factor)
Definition: vpKeyPoint.h:969
cv::Ptr< cv::FeatureDetector > getDetector(const vpFeatureDetectorType &type) const
Definition: vpKeyPoint.h:489
void setExtractor(const vpFeatureDescriptorType &extractorType)
Definition: vpKeyPoint.h:837
std::vector< cv::DMatch > getMatches() const
Definition: vpKeyPoint.h:614
void setRansacReprojectionError(const double reprojectionError)
Definition: vpKeyPoint.h:1059
void setMatchingRatioThreshold(const double ratio)
Definition: vpKeyPoint.h:983
void setCovarianceComputation(const bool &flag)
Definition: vpKeyPoint.h:753
RANSAC_FILTER_FLAGS
Definition: vpPose.h:101