Visual Servoing Platform  version 3.0.1
 All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
vpFernClassifier.cpp
1 /****************************************************************************
2  *
3  * This file is part of the ViSP software.
4  * Copyright (C) 2005 - 2017 by Inria. All rights reserved.
5  *
6  * This software is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * ("GPL") version 2 as published by the Free Software Foundation.
9  * See the file LICENSE.txt at the root directory of this source
10  * distribution for additional information about the GNU GPL.
11  *
12  * For using ViSP with software that can not be combined with the GNU
13  * GPL, please contact Inria about acquiring a ViSP Professional
14  * Edition License.
15  *
16  * See http://visp.inria.fr for more information.
17  *
18  * This software was developed at:
19  * Inria Rennes - Bretagne Atlantique
20  * Campus Universitaire de Beaulieu
21  * 35042 Rennes Cedex
22  * France
23  *
24  * If you have questions regarding the use of this file, please contact
25  * Inria at visp@inria.fr
26  *
27  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
28  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
29  *
30  * Description:
31  * Class that implements the Fern classifier and the YAPE detector thanks
32  * to the OpenCV library.
33  *
34  * Authors:
35  * Romain Tallonneau
36  *
37  *****************************************************************************/
38 
39 #include <visp3/core/vpConfig.h>
40 
41 #if (VISP_HAVE_OPENCV_VERSION >= 0x020000) && (VISP_HAVE_OPENCV_VERSION < 0x030000) // Require opencv >= 2.0.0 and < 3.0.0
42 
43 #include <visp3/vision/vpFernClassifier.h>
44 #include <visp3/core/vpImageTools.h>
45 #include <visp3/core/vpImageConvert.h>
46 #include <visp3/core/vpColor.h>
47 #include <visp3/core/vpDisplay.h>
48 
54  : vpBasicKeyPoint(),
55  ldetector(), fernClassifier(), gen(0, 256, 5, true, 0.6, 1.5, -CV_PI/2, CV_PI/2, -CV_PI/2, CV_PI/2),
56  hasLearn(false), threshold(20), nbView(2000), dist(2), nbClassfier(100), ClassifierSize(11),
57  nbOctave(2), patchSize(32), radius(7), nbPoints(200), blurImage(true), radiusBlur(7),
58  sigmaBlur(1), nbMinPoint(10),
59  #if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
60  curImg(),
61  #else
62  curImg(NULL),
63  #endif
64  objKeypoints(), modelROI_Ref(), modelROI(),
65  modelPoints(), imgKeypoints(), refPt(), curPt()
66 {
67 }
68 
79 vpFernClassifier::vpFernClassifier(const std::string& _dataFile, const std::string& _objectName)
80  : vpBasicKeyPoint(),
81  ldetector(), fernClassifier(), gen(0, 256, 5, true, 0.6, 1.5, -CV_PI/2, CV_PI/2, -CV_PI/2, CV_PI/2),
82  hasLearn(false), threshold(20), nbView(2000), dist(2), nbClassfier(100), ClassifierSize(11),
83  nbOctave(2), patchSize(32), radius(7), nbPoints(200), blurImage(true), radiusBlur(7),
84  sigmaBlur(1), nbMinPoint(10),
85  #if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
86  curImg(),
87  #else
88  curImg(NULL),
89  #endif
90  objKeypoints(), modelROI_Ref(), modelROI(),
91  modelPoints(), imgKeypoints(), refPt(), curPt()
92 {
93  this->load(_dataFile, _objectName);
94 }
95 
101 {
102 #if (VISP_HAVE_OPENCV_VERSION < 0x020408)
103  if(curImg != NULL){
104  if(curImg->width % 8 == 0){
105  curImg->imageData = NULL;
106  cvReleaseImageHeader(&curImg);
107  }else{
108  cvReleaseImage(&curImg);
109  }
110  curImg = NULL;
111  }
112 #endif
113 }
114 
120 void
122 {
123  hasLearn = false;
124  nbClassfier = 100;
125  ClassifierSize = 11;
126  nbPoints = 200;
127 #if (VISP_HAVE_OPENCV_VERSION < 0x020408)
128  curImg = NULL;
129 #endif
130  blurImage = true;
131  radiusBlur = 7;
132  sigmaBlur = 1;
133  patchSize = 32;
134  radius = 7;
135  threshold = 20;
136  nbOctave = 2;
137  nbView = 2000;
138  dist = 2;
139  nbMinPoint = 10;
140 }
141 
142 
143 
144 
148 void
150 {
151  // initialise detector
152  cv::LDetector d(radius, threshold, nbOctave, nbView, patchSize, dist);
153 
154  //blur
155  cv::Mat obj = (cv::Mat)curImg;
156 
157  if(this->getBlurSetting()){
158  cv::GaussianBlur(obj, obj, cv::Size(getBlurSize(), getBlurSize()), getBlurSigma(), getBlurSigma());
159  }
160 
161  // build pyramid
162  std::vector<cv::Mat> objpyr;
163  cv::buildPyramid(obj, objpyr, d.nOctaves-1);
164 
165  // getPoints
166  d.getMostStable2D(obj, objKeypoints, 100, gen);
167 
168  ldetector = d;
169 
170  // train classifier
171  modelROI = cv::Rect(0, 0, objpyr[0].cols, objpyr[0].rows);
172  ldetector.getMostStable2D(objpyr[0], modelPoints, 100, gen);
173 
174  fernClassifier.trainFromSingleView(objpyr[0], modelPoints,
175  patchSize, (int)modelPoints.size(), 100, 11, 10000,
176  cv::FernClassifier::COMPRESSION_NONE, gen);
177 
178  /* from OpenCV format to ViSP format */
179  referenceImagePointsList.resize(0);
180  for (unsigned int i = 0; i < modelPoints.size(); i += 1){
181  vpImagePoint ip(
182  modelPoints[i].pt.y + modelROI_Ref.y,
183  modelPoints[i].pt.x + modelROI_Ref.x);
184  referenceImagePointsList.push_back(ip);
185  }
186 
187  // set flag
188  hasLearn = true;
189 }
190 
204 unsigned int
206 {
207  this->setImage(_I);
208 
209  train();
210 
211  _reference_computed = true;
212  return (unsigned int)objKeypoints.size();
213 }
214 
233 unsigned int
235  const vpImagePoint &_iP,
236  const unsigned int _height, const unsigned int _width)
237 {
238  if((_iP.get_i()+_height) >= _I.getHeight()
239  || (_iP.get_j()+_width) >= _I.getWidth()) {
240  vpTRACE("Bad size for the subimage");
242  "Bad size for the subimage"));
243  }
244 
245  vpImage<unsigned char> subImage;
247  (unsigned int)_iP.get_i(),
248  (unsigned int)_iP.get_j(),
249  _height, _width, subImage);
250  this->setImage(subImage);
251 
252  /* initialise a structure containing the region of interest used in the
253  reference image */
254  modelROI_Ref.x = (int)_iP.get_u();
255  modelROI_Ref.y = (int)_iP.get_v();
256  modelROI_Ref.width = (int)_width;
257  modelROI_Ref.height = (int)_height;
258 
259  train();
260 
261  return (unsigned int)objKeypoints.size();
262 }
263 
280 unsigned int
282  const vpRect& _rectangle)
283 {
284  vpImagePoint iP;
285  iP.set_i(_rectangle.getTop());
286  iP.set_j(_rectangle.getLeft());
287  return (this->buildReference(_I, iP,
288  (unsigned int)_rectangle.getHeight(),
289  (unsigned int)_rectangle.getWidth()));
290 }
291 
303 unsigned int
305 {
306  if(!hasLearn){
307  vpERROR_TRACE("The object has not been learned. ");
308  throw vpException(vpException::notInitialized , "object is not learned, load database or build the reference ");
309  }
310 
311  setImage(_I);
312  // resize image
313 // cv::resize(_image, image, Size(), 1./imgscale, 1./imgscale, INTER_CUBIC);
314  cv::Mat img = this->curImg;
315 
316  if(this->getBlurSetting()){
317  cv::GaussianBlur(img, img, cv::Size(this->getBlurSize(), this->getBlurSize()), this->getBlurSigma(), this->getBlurSigma());
318  }
319 
320  std::vector<cv::Mat> imgPyr;
321  cv::buildPyramid(img, imgPyr, ldetector.nOctaves-1);
322 
323 
324  ldetector(imgPyr, imgKeypoints, 500);
325 
326  unsigned int m = (unsigned int)modelPoints.size();
327  unsigned int n = (unsigned int)imgKeypoints.size();
328  std::vector<int> bestMatches(m, -1);
329  std::vector<float> maxLogProb(m, -FLT_MAX);
330  std::vector<float> signature;
331  unsigned int totalMatch = 0;
332 
333 
334  /* part of code from OpenCV planarObjectDetector */
335  currentImagePointsList.resize(0);
336  matchedReferencePoints.resize(0);
337 
338  for(unsigned int i = 0; i < n; i++ ){
339  cv::KeyPoint kpt = imgKeypoints[i];
340  kpt.pt.x /= (float)(1 << kpt.octave);
341  kpt.pt.y /= (float)(1 << kpt.octave);
342  int k = fernClassifier(imgPyr[(unsigned int)kpt.octave], kpt.pt, signature);
343  if( k >= 0 && (bestMatches[(unsigned int)k] < 0 || signature[(unsigned int)k] > maxLogProb[(unsigned int)k]) ){
344  maxLogProb[(unsigned int)k] = signature[(unsigned int)k];
345  bestMatches[(unsigned int)k] = (int)i;
346  totalMatch++;
347 
348  vpImagePoint ip_cur( imgKeypoints[i].pt.y, imgKeypoints[i].pt.x);
349 
350  currentImagePointsList.push_back(ip_cur);
351  matchedReferencePoints.push_back((unsigned int)k);
352 
353  }
354  }
355 
356  refPt.resize(0);
357  curPt.resize(0);
358  for(unsigned int i = 0; i < m; i++ ){
359  if( bestMatches[i] >= 0 ){
360  refPt.push_back(modelPoints[i].pt);
361  curPt.push_back(imgKeypoints[(unsigned int)bestMatches[i]].pt);
362  }
363  }
364 
365  return totalMatch;
366 }
367 
368 
369 
383 unsigned int
385  const vpImagePoint &_iP,
386  const unsigned int _height, const unsigned int _width)
387 {
388  if((_iP.get_i()+_height) >= _I.getHeight()
389  || (_iP.get_j()+_width) >= _I.getWidth()) {
390  vpTRACE("Bad size for the subimage");
392  "Bad size for the subimage"));
393  }
394 
395  vpImage<unsigned char> subImage;
396 
398  (unsigned int)_iP.get_i(),
399  (unsigned int)_iP.get_j(),
400  _height, _width, subImage);
401 
402  return this->matchPoint(subImage);
403 }
404 
416 unsigned int
418  const vpRect& _rectangle)
419 {
420  vpImagePoint iP;
421  iP.set_i(_rectangle.getTop());
422  iP.set_j(_rectangle.getLeft());
423  return (this->matchPoint(_I, iP,
424  (unsigned int)_rectangle.getHeight(),
425  (unsigned int)_rectangle.getWidth()));
426 }
427 
428 
445 void
447  const vpImage<unsigned char> &_Icurrent, unsigned int size)
448 {
449  for (unsigned int i = 0; i < matchedReferencePoints.size(); i++){
452  }
453 
454 }
455 
456 
467 void
468 vpFernClassifier::display(const vpImage<unsigned char> &_Icurrent, unsigned int size,
469  const vpColor &color)
470 {
471  for (unsigned int i = 0; i < matchedReferencePoints.size(); i++){
472  vpDisplay::displayCross (_Icurrent, currentImagePointsList[i], size, color);
473  }
474 }
475 
476 /* IO METHODS */
477 
486 void
487 vpFernClassifier::load(const std::string& _dataFile, const std::string& /*_objectName*/)
488 {
489  std::cout << " > Load data for the planar object detector..." << std::endl;
490 
491  /* part of code from OpenCV planarObjectDetector */
492  cv::FileStorage fs(_dataFile, cv::FileStorage::READ);
493  cv::FileNode node = fs.getFirstTopLevelNode();
494 
495  cv::FileNodeIterator it = node["model-roi"].begin(), it_end;
496  it >> modelROI.x >> modelROI.y >> modelROI.width >> modelROI.height;
497 
498  ldetector.read(node["detector"]);
499  fernClassifier.read(node["fern-classifier"]);
500 
501  const cv::FileNode node_ = node["model-points"];
502  cv::read(node_, modelPoints);
503 
504  cv::LDetector d(radius, threshold, nbOctave, nbView, patchSize, dist);
505  ldetector = d;
506  hasLearn = true;
507 }
508 
509 
516 void
517 vpFernClassifier::record(const std::string& _objectName, const std::string& _dataFile )
518 {
519  /* part of code from OpenCV planarObjectDetector */
520  cv::FileStorage fs(_dataFile, cv::FileStorage::WRITE);
521 
522  cv::WriteStructContext ws(fs, _objectName, CV_NODE_MAP);
523 
524  {
525  cv::WriteStructContext wsroi(fs, "model-roi", CV_NODE_SEQ + CV_NODE_FLOW);
526  cv::write(fs, modelROI_Ref.x);
527  cv::write(fs, modelROI_Ref.y);
528  cv::write(fs, modelROI_Ref.width);
529  cv::write(fs, modelROI_Ref.height);
530  }
531 
532  ldetector.write(fs, "detector");
533  cv::write(fs, "model-points", modelPoints);
534  fernClassifier.write(fs, "fern-classifier");
535 }
536 
537 
538 
545 void
547 {
548 #if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
550 #else
551  if(curImg != NULL){
552  cvResetImageROI(curImg);
553  if((curImg->width % 8) == 0){
554  curImg->imageData = NULL;
555  cvReleaseImageHeader(&curImg);
556  }else{
557  cvReleaseImage(&curImg);
558  }
559  curImg = NULL;
560  }
561  if((I.getWidth()%8) == 0){
562  curImg = cvCreateImageHeader(cvSize((int)I.getWidth(), (int)I.getHeight()), IPL_DEPTH_8U, 1);
563  if(curImg != NULL){
564  curImg->imageData = (char*)I.bitmap;
565  }else{
566  throw vpException(vpException::memoryAllocationError, "Could not create the image in the OpenCV format.");
567  }
568  }else{
570  }
571  if(curImg == NULL){
572  std::cout << "!> conversion failed" << std::endl;
573  throw vpException(vpException::notInitialized , "conversion failed");
574  }
575 #endif
576 
577 }
578 
579 #elif !defined(VISP_BUILD_SHARED_LIBS)
580 // Work arround to avoid warning: libvisp_vision.a(vpFernClassifier.cpp.o) has no symbols
581 void dummy_vpFernClassifier() {};
582 #endif
583 
virtual void init()
class that defines what is a Keypoint. This class provides all the basic elements to implement classe...
double getTop() const
Definition: vpRect.h:178
double get_v() const
Definition: vpImagePoint.h:268
std::vector< cv::KeyPoint > imgKeypoints
the vector containing the points in the current image.
bool blurImage
Flag to specify whether the reference image have to be blurred or not in order to improve the recogni...
double get_i() const
Definition: vpImagePoint.h:199
unsigned int getWidth() const
Definition: vpImage.h:226
bool hasLearn
Flag to indicate whether the classifier has been trained or not.
static void convert(const vpImage< unsigned char > &src, vpImage< vpRGBa > &dest)
virtual unsigned int matchPoint(const vpImage< unsigned char > &I)
Type * bitmap
points toward the bitmap
Definition: vpImage.h:134
#define vpERROR_TRACE
Definition: vpDebug.h:391
Class to define colors available for display functionnalities.
Definition: vpColor.h:121
int nbClassfier
Number of classifier.
int nbView
Number of view to generate for the training.
cv::Rect modelROI_Ref
the ROI in the reference image.
double get_u() const
Definition: vpImagePoint.h:257
cv::PatchGenerator gen
The patch generator (OpenCV format).
unsigned int nbMinPoint
Number of minimum point below which the homography is not estimated (must be at least four) ...
error that can be emited by ViSP classes.
Definition: vpException.h:73
double getHeight() const
Definition: vpRect.h:152
cv::LDetector ldetector
The points of interest detector.
static const vpColor green
Definition: vpColor.h:166
int radius
Radius for the detector.
int nbPoints
Maximal number of points.
double get_j() const
Definition: vpImagePoint.h:210
static const vpColor red
Definition: vpColor.h:163
std::vector< cv::Point2f > curPt
virtual unsigned int buildReference(const vpImage< unsigned char > &I)
int patchSize
Size of the patch.
void set_i(const double ii)
Definition: vpImagePoint.h:163
double getWidth() const
Definition: vpRect.h:199
int sigmaBlur
Sigma of the kernel used to blur the image.
int nbOctave
Number of octave for the multi scale.
#define vpTRACE
Definition: vpDebug.h:414
bool _reference_computed
flag to indicate if the reference has been built.
std::vector< vpImagePoint > referenceImagePointsList
void record(const std::string &_objectName, const std::string &_dataFile)
record the Ferns classifier in the text file
std::vector< cv::KeyPoint > objKeypoints
keypoints detected in the reference image.
void set_j(const double jj)
Definition: vpImagePoint.h:174
static void displayCross(const vpImage< unsigned char > &I, const vpImagePoint &ip, unsigned int size, const vpColor &color, unsigned int thickness=1)
int threshold
Threshold to accept or reject points (usually around 20)
static void crop(const vpImage< Type > &I, double roi_top, double roi_left, unsigned int roi_height, unsigned int roi_width, vpImage< Type > &crop, unsigned int v_scale=1, unsigned int h_scale=1)
Definition: vpImageTools.h:266
cv::FernClassifier fernClassifier
The Fern classifier.
int dist
Minimal distance between two points.
std::vector< unsigned int > matchedReferencePoints
std::vector< cv::Point2f > refPt
vector in the OpenCV format to be used by the detector.
unsigned int getHeight() const
Definition: vpImage.h:175
Defines a rectangle in the plane.
Definition: vpRect.h:82
int radiusBlur
Radius of the kernel used to blur the image.
std::vector< vpImagePoint > currentImagePointsList
cv::Mat curImg
The current image in the OpenCV format.
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
Definition: vpImagePoint.h:88
void load(const std::string &_dataFile, const std::string &)
load the Fern classifier
virtual void display(const vpImage< unsigned char > &Iref, const vpImage< unsigned char > &Icurrent, unsigned int size=3)
void setImage(const vpImage< unsigned char > &I)
std::vector< cv::KeyPoint > modelPoints
the vector containing the points in the model.
double getLeft() const
Definition: vpRect.h:159
int ClassifierSize
Size of the classifier.
cv::Rect modelROI
the ROI for the reference image.