Visual Servoing Platform  version 3.1.0
vpFernClassifier.cpp
1 /****************************************************************************
2  *
3  * This file is part of the ViSP software.
4  * Copyright (C) 2005 - 2017 by Inria. All rights reserved.
5  *
6  * This software is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  * See the file LICENSE.txt at the root directory of this source
11  * distribution for additional information about the GNU GPL.
12  *
13  * For using ViSP with software that can not be combined with the GNU
14  * GPL, please contact Inria about acquiring a ViSP Professional
15  * Edition License.
16  *
17  * See http://visp.inria.fr for more information.
18  *
19  * This software was developed at:
20  * Inria Rennes - Bretagne Atlantique
21  * Campus Universitaire de Beaulieu
22  * 35042 Rennes Cedex
23  * France
24  *
25  * If you have questions regarding the use of this file, please contact
26  * Inria at visp@inria.fr
27  *
28  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
29  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
30  *
31  * Description:
32  * Class that implements the Fern classifier and the YAPE detector thanks
33  * to the OpenCV library.
34  *
35  * Authors:
36  * Romain Tallonneau
37  *
38  *****************************************************************************/
39 
40 #include <visp3/core/vpConfig.h>
41 
42 #if (VISP_HAVE_OPENCV_VERSION >= 0x020000) && \
43  (VISP_HAVE_OPENCV_VERSION < 0x030000) // Require opencv >= 2.0.0 and < 3.0.0
44 
45 #include <visp3/core/vpColor.h>
46 #include <visp3/core/vpDisplay.h>
47 #include <visp3/core/vpImageConvert.h>
48 #include <visp3/core/vpImageTools.h>
49 #include <visp3/vision/vpFernClassifier.h>
50 
56  : vpBasicKeyPoint(), ldetector(), fernClassifier(),
57  gen(0, 256, 5, true, 0.6, 1.5, -CV_PI / 2, CV_PI / 2, -CV_PI / 2, CV_PI / 2), hasLearn(false), threshold(20),
58  nbView(2000), dist(2), nbClassfier(100), ClassifierSize(11), nbOctave(2), patchSize(32), radius(7), nbPoints(200),
59  blurImage(true), radiusBlur(7), sigmaBlur(1), nbMinPoint(10),
60 #if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
61  curImg(),
62 #else
63  curImg(NULL),
64 #endif
65  objKeypoints(), modelROI_Ref(), modelROI(), modelPoints(), imgKeypoints(), refPt(), curPt()
66 {
67 }
68 
80 vpFernClassifier::vpFernClassifier(const std::string &_dataFile, const std::string &_objectName)
82  gen(0, 256, 5, true, 0.6, 1.5, -CV_PI / 2, CV_PI / 2, -CV_PI / 2, CV_PI / 2), hasLearn(false), threshold(20),
83  nbView(2000), dist(2), nbClassfier(100), ClassifierSize(11), nbOctave(2), patchSize(32), radius(7), nbPoints(200),
84  blurImage(true), radiusBlur(7), sigmaBlur(1), nbMinPoint(10),
85 #if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
86  curImg(),
87 #else
88  curImg(NULL),
89 #endif
91 {
92  this->load(_dataFile, _objectName);
93 }
94 
100 {
101 #if (VISP_HAVE_OPENCV_VERSION < 0x020408)
102  if (curImg != NULL) {
103  if (curImg->width % 8 == 0) {
104  curImg->imageData = NULL;
105  cvReleaseImageHeader(&curImg);
106  } else {
107  cvReleaseImage(&curImg);
108  }
109  curImg = NULL;
110  }
111 #endif
112 }
113 
121 {
122  hasLearn = false;
123  nbClassfier = 100;
124  ClassifierSize = 11;
125  nbPoints = 200;
126 #if (VISP_HAVE_OPENCV_VERSION < 0x020408)
127  curImg = NULL;
128 #endif
129  blurImage = true;
130  radiusBlur = 7;
131  sigmaBlur = 1;
132  patchSize = 32;
133  radius = 7;
134  threshold = 20;
135  nbOctave = 2;
136  nbView = 2000;
137  dist = 2;
138  nbMinPoint = 10;
139 }
140 
145 {
146  // initialise detector
147  cv::LDetector d(radius, threshold, nbOctave, nbView, patchSize, dist);
148 
149  // blur
150  cv::Mat obj = (cv::Mat)curImg;
151 
152  if (this->getBlurSetting()) {
153  cv::GaussianBlur(obj, obj, cv::Size(getBlurSize(), getBlurSize()), getBlurSigma(), getBlurSigma());
154  }
155 
156  // build pyramid
157  std::vector<cv::Mat> objpyr;
158  cv::buildPyramid(obj, objpyr, d.nOctaves - 1);
159 
160  // getPoints
161  d.getMostStable2D(obj, objKeypoints, 100, gen);
162 
163  ldetector = d;
164 
165  // train classifier
166  modelROI = cv::Rect(0, 0, objpyr[0].cols, objpyr[0].rows);
167  ldetector.getMostStable2D(objpyr[0], modelPoints, 100, gen);
168 
169  fernClassifier.trainFromSingleView(objpyr[0], modelPoints, patchSize, (int)modelPoints.size(), 100, 11, 10000,
170  cv::FernClassifier::COMPRESSION_NONE, gen);
171 
172  /* from OpenCV format to ViSP format */
173  referenceImagePointsList.resize(0);
174  for (unsigned int i = 0; i < modelPoints.size(); i += 1) {
175  vpImagePoint ip(modelPoints[i].pt.y + modelROI_Ref.y, modelPoints[i].pt.x + modelROI_Ref.x);
176  referenceImagePointsList.push_back(ip);
177  }
178 
179  // set flag
180  hasLearn = true;
181 }
182 
197 {
198  this->setImage(_I);
199 
200  train();
201 
202  _reference_computed = true;
203  return (unsigned int)objKeypoints.size();
204 }
205 
225  const unsigned int _height, const unsigned int _width)
226 {
227  if ((_iP.get_i() + _height) >= _I.getHeight() || (_iP.get_j() + _width) >= _I.getWidth()) {
228  vpTRACE("Bad size for the subimage");
229  throw(vpException(vpImageException::notInTheImage, "Bad size for the subimage"));
230  }
231 
232  vpImage<unsigned char> subImage;
233  vpImageTools::crop(_I, (unsigned int)_iP.get_i(), (unsigned int)_iP.get_j(), _height, _width, subImage);
234  this->setImage(subImage);
235 
236  /* initialise a structure containing the region of interest used in the
237  reference image */
238  modelROI_Ref.x = (int)_iP.get_u();
239  modelROI_Ref.y = (int)_iP.get_v();
240  modelROI_Ref.width = (int)_width;
241  modelROI_Ref.height = (int)_height;
242 
243  train();
244 
245  return (unsigned int)objKeypoints.size();
246 }
247 
264 unsigned int vpFernClassifier::buildReference(const vpImage<unsigned char> &_I, const vpRect &_rectangle)
265 {
266  vpImagePoint iP;
267  iP.set_i(_rectangle.getTop());
268  iP.set_j(_rectangle.getLeft());
269  return (this->buildReference(_I, iP, (unsigned int)_rectangle.getHeight(), (unsigned int)_rectangle.getWidth()));
270 }
271 
285 {
286  if (!hasLearn) {
287  vpERROR_TRACE("The object has not been learned. ");
288  throw vpException(vpException::notInitialized, "object is not learned, load database or build the reference ");
289  }
290 
291  setImage(_I);
292  // resize image
293  // cv::resize(_image, image, Size(), 1./imgscale, 1./imgscale,
294  // INTER_CUBIC);
295  cv::Mat img = this->curImg;
296 
297  if (this->getBlurSetting()) {
298  cv::GaussianBlur(img, img, cv::Size(this->getBlurSize(), this->getBlurSize()), this->getBlurSigma(),
299  this->getBlurSigma());
300  }
301 
302  std::vector<cv::Mat> imgPyr;
303  cv::buildPyramid(img, imgPyr, ldetector.nOctaves - 1);
304 
305  ldetector(imgPyr, imgKeypoints, 500);
306 
307  unsigned int m = (unsigned int)modelPoints.size();
308  unsigned int n = (unsigned int)imgKeypoints.size();
309  std::vector<int> bestMatches(m, -1);
310  std::vector<float> maxLogProb(m, -FLT_MAX);
311  std::vector<float> signature;
312  unsigned int totalMatch = 0;
313 
314  /* part of code from OpenCV planarObjectDetector */
315  currentImagePointsList.resize(0);
316  matchedReferencePoints.resize(0);
317 
318  for (unsigned int i = 0; i < n; i++) {
319  cv::KeyPoint kpt = imgKeypoints[i];
320  kpt.pt.x /= (float)(1 << kpt.octave);
321  kpt.pt.y /= (float)(1 << kpt.octave);
322  int k = fernClassifier(imgPyr[(unsigned int)kpt.octave], kpt.pt, signature);
323  if (k >= 0 && (bestMatches[(unsigned int)k] < 0 || signature[(unsigned int)k] > maxLogProb[(unsigned int)k])) {
324  maxLogProb[(unsigned int)k] = signature[(unsigned int)k];
325  bestMatches[(unsigned int)k] = (int)i;
326  totalMatch++;
327 
328  vpImagePoint ip_cur(imgKeypoints[i].pt.y, imgKeypoints[i].pt.x);
329 
330  currentImagePointsList.push_back(ip_cur);
331  matchedReferencePoints.push_back((unsigned int)k);
332  }
333  }
334 
335  refPt.resize(0);
336  curPt.resize(0);
337  for (unsigned int i = 0; i < m; i++) {
338  if (bestMatches[i] >= 0) {
339  refPt.push_back(modelPoints[i].pt);
340  curPt.push_back(imgKeypoints[(unsigned int)bestMatches[i]].pt);
341  }
342  }
343 
344  return totalMatch;
345 }
346 
361  const unsigned int _height, const unsigned int _width)
362 {
363  if ((_iP.get_i() + _height) >= _I.getHeight() || (_iP.get_j() + _width) >= _I.getWidth()) {
364  vpTRACE("Bad size for the subimage");
365  throw(vpException(vpImageException::notInTheImage, "Bad size for the subimage"));
366  }
367 
368  vpImage<unsigned char> subImage;
369 
370  vpImageTools::crop(_I, (unsigned int)_iP.get_i(), (unsigned int)_iP.get_j(), _height, _width, subImage);
371 
372  return this->matchPoint(subImage);
373 }
374 
386 unsigned int vpFernClassifier::matchPoint(const vpImage<unsigned char> &_I, const vpRect &_rectangle)
387 {
388  vpImagePoint iP;
389  iP.set_i(_rectangle.getTop());
390  iP.set_j(_rectangle.getLeft());
391  return (this->matchPoint(_I, iP, (unsigned int)_rectangle.getHeight(), (unsigned int)_rectangle.getWidth()));
392 }
393 
412  unsigned int size)
413 {
414  for (unsigned int i = 0; i < matchedReferencePoints.size(); i++) {
417  }
418 }
419 
431 void vpFernClassifier::display(const vpImage<unsigned char> &_Icurrent, unsigned int size, const vpColor &color)
432 {
433  for (unsigned int i = 0; i < matchedReferencePoints.size(); i++) {
434  vpDisplay::displayCross(_Icurrent, currentImagePointsList[i], size, color);
435  }
436 }
437 
438 /* IO METHODS */
439 
449 void vpFernClassifier::load(const std::string &_dataFile, const std::string & /*_objectName*/)
450 {
451  std::cout << " > Load data for the planar object detector..." << std::endl;
452 
453  /* part of code from OpenCV planarObjectDetector */
454  cv::FileStorage fs(_dataFile, cv::FileStorage::READ);
455  cv::FileNode node = fs.getFirstTopLevelNode();
456 
457  cv::FileNodeIterator it = node["model-roi"].begin(), it_end;
458  it >> modelROI.x >> modelROI.y >> modelROI.width >> modelROI.height;
459 
460  ldetector.read(node["detector"]);
461  fernClassifier.read(node["fern-classifier"]);
462 
463  const cv::FileNode node_ = node["model-points"];
464  cv::read(node_, modelPoints);
465 
466  cv::LDetector d(radius, threshold, nbOctave, nbView, patchSize, dist);
467  ldetector = d;
468  hasLearn = true;
469 }
470 
478 void vpFernClassifier::record(const std::string &_objectName, const std::string &_dataFile)
479 {
480  /* part of code from OpenCV planarObjectDetector */
481  cv::FileStorage fs(_dataFile, cv::FileStorage::WRITE);
482 
483  cv::WriteStructContext ws(fs, _objectName, CV_NODE_MAP);
484 
485  {
486  cv::WriteStructContext wsroi(fs, "model-roi", CV_NODE_SEQ + CV_NODE_FLOW);
487  cv::write(fs, modelROI_Ref.x);
488  cv::write(fs, modelROI_Ref.y);
489  cv::write(fs, modelROI_Ref.width);
490  cv::write(fs, modelROI_Ref.height);
491  }
492 
493  ldetector.write(fs, "detector");
494  cv::write(fs, "model-points", modelPoints);
495  fernClassifier.write(fs, "fern-classifier");
496 }
497 
505 {
506 #if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
508 #else
509  if (curImg != NULL) {
510  cvResetImageROI(curImg);
511  if ((curImg->width % 8) == 0) {
512  curImg->imageData = NULL;
513  cvReleaseImageHeader(&curImg);
514  } else {
515  cvReleaseImage(&curImg);
516  }
517  curImg = NULL;
518  }
519  if ((I.getWidth() % 8) == 0) {
520  curImg = cvCreateImageHeader(cvSize((int)I.getWidth(), (int)I.getHeight()), IPL_DEPTH_8U, 1);
521  if (curImg != NULL) {
522  curImg->imageData = (char *)I.bitmap;
523  } else {
524  throw vpException(vpException::memoryAllocationError, "Could not create the image in the OpenCV format.");
525  }
526  } else {
528  }
529  if (curImg == NULL) {
530  std::cout << "!> conversion failed" << std::endl;
531  throw vpException(vpException::notInitialized, "conversion failed");
532  }
533 #endif
534 }
535 
536 #elif !defined(VISP_BUILD_SHARED_LIBS)
537 // Work arround to avoid warning: libvisp_vision.a(vpFernClassifier.cpp.o) has
538 // no symbols
539 void dummy_vpFernClassifier(){};
540 #endif
virtual void init()
double get_i() const
Definition: vpImagePoint.h:204
class that defines what is a Keypoint. This class provides all the basic elements to implement classe...
double getTop() const
Definition: vpRect.h:175
virtual ~vpFernClassifier()
std::vector< cv::KeyPoint > imgKeypoints
the vector containing the points in the current image.
bool hasLearn
Flag to indicate whether the classifier has been trained or not.
static void convert(const vpImage< unsigned char > &src, vpImage< vpRGBa > &dest)
virtual unsigned int matchPoint(const vpImage< unsigned char > &I)
Type * bitmap
points toward the bitmap
Definition: vpImage.h:133
#define vpERROR_TRACE
Definition: vpDebug.h:393
Class to define colors available for display functionnalities.
Definition: vpColor.h:120
int nbClassfier
Number of classifier.
int nbView
Number of view to generate for the training.
cv::Rect modelROI_Ref
the ROI in the reference image.
cv::PatchGenerator gen
The patch generator (OpenCV format).
unsigned int nbMinPoint
error that can be emited by ViSP classes.
Definition: vpException.h:71
cv::LDetector ldetector
The points of interest detector.
static const vpColor green
Definition: vpColor.h:183
int radius
Radius for the detector.
int nbPoints
Maximal number of points.
static const vpColor red
Definition: vpColor.h:180
std::vector< cv::Point2f > curPt
virtual unsigned int buildReference(const vpImage< unsigned char > &I)
int patchSize
Size of the patch.
void set_i(const double ii)
Definition: vpImagePoint.h:167
double get_u() const
Definition: vpImagePoint.h:263
double getWidth() const
Definition: vpRect.h:197
int sigmaBlur
Sigma of the kernel used to blur the image.
int nbOctave
Number of octave for the multi scale.
#define vpTRACE
Definition: vpDebug.h:416
bool _reference_computed
flag to indicate if the reference has been built.
double get_j() const
Definition: vpImagePoint.h:215
double getLeft() const
Definition: vpRect.h:156
std::vector< vpImagePoint > referenceImagePointsList
void record(const std::string &_objectName, const std::string &_dataFile)
record the Ferns classifier in the text file
std::vector< cv::KeyPoint > objKeypoints
keypoints detected in the reference image.
void set_j(const double jj)
Definition: vpImagePoint.h:178
static void displayCross(const vpImage< unsigned char > &I, const vpImagePoint &ip, unsigned int size, const vpColor &color, unsigned int thickness=1)
int threshold
Threshold to accept or reject points (usually around 20)
unsigned int getHeight() const
Definition: vpImage.h:178
static void crop(const vpImage< Type > &I, double roi_top, double roi_left, unsigned int roi_height, unsigned int roi_width, vpImage< Type > &crop, unsigned int v_scale=1, unsigned int h_scale=1)
Definition: vpImageTools.h:237
cv::FernClassifier fernClassifier
The Fern classifier.
int dist
Minimal distance between two points.
double getHeight() const
Definition: vpRect.h:149
std::vector< unsigned int > matchedReferencePoints
std::vector< cv::Point2f > refPt
vector in the OpenCV format to be used by the detector.
Defines a rectangle in the plane.
Definition: vpRect.h:78
int radiusBlur
Radius of the kernel used to blur the image.
std::vector< vpImagePoint > currentImagePointsList
cv::Mat curImg
The current image in the OpenCV format.
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
Definition: vpImagePoint.h:88
void load(const std::string &_dataFile, const std::string &)
load the Fern classifier
virtual void display(const vpImage< unsigned char > &Iref, const vpImage< unsigned char > &Icurrent, unsigned int size=3)
void setImage(const vpImage< unsigned char > &I)
std::vector< cv::KeyPoint > modelPoints
the vector containing the points in the model.
unsigned int getWidth() const
Definition: vpImage.h:229
int ClassifierSize
Size of the classifier.
double get_v() const
Definition: vpImagePoint.h:274
cv::Rect modelROI
the ROI for the reference image.