Visual Servoing Platform  version 3.6.1 under development (2023-10-19)
vpFernClassifier.cpp
1 /*
2  * ViSP, open source Visual Servoing Platform software.
3  * Copyright (C) 2005 - 2023 by Inria. All rights reserved.
4  *
5  * This software is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  * See the file LICENSE.txt at the root directory of this source
10  * distribution for additional information about the GNU GPL.
11  *
12  * For using ViSP with software that can not be combined with the GNU
13  * GPL, please contact Inria about acquiring a ViSP Professional
14  * Edition License.
15  *
16  * See https://visp.inria.fr for more information.
17  *
18  * This software was developed at:
19  * Inria Rennes - Bretagne Atlantique
20  * Campus Universitaire de Beaulieu
21  * 35042 Rennes Cedex
22  * France
23  *
24  * If you have questions regarding the use of this file, please contact
25  * Inria at visp@inria.fr
26  *
27  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
28  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
29  *
30  * Description:
31  * Class that implements the Fern classifier and the YAPE detector thanks
32  * to the OpenCV library.
33  */
34 
35 #include <visp3/core/vpConfig.h>
36 
37 #if (VISP_HAVE_OPENCV_VERSION >= 0x020408) && \
38  (VISP_HAVE_OPENCV_VERSION < 0x030000) // Require opencv >= 2.4.8 and < 3.0.0
39 
40 #include <visp3/core/vpColor.h>
41 #include <visp3/core/vpDisplay.h>
42 #include <visp3/core/vpImageConvert.h>
43 #include <visp3/core/vpImageTools.h>
44 #include <visp3/vision/vpFernClassifier.h>
45 
50 vpFernClassifier::vpFernClassifier()
51  : vpBasicKeyPoint(), ldetector(), fernClassifier(),
52  gen(0, 256, 5, true, 0.6, 1.5, -CV_PI / 2, CV_PI / 2, -CV_PI / 2, CV_PI / 2), hasLearn(false), threshold(20),
53  nbView(2000), dist(2), nbClassfier(100), ClassifierSize(11), nbOctave(2), patchSize(32), radius(7), nbPoints(200),
54  blurImage(true), radiusBlur(7), sigmaBlur(1), nbMinPoint(10),
55  curImg(), objKeypoints(), modelROI_Ref(), modelROI(), modelPoints(), imgKeypoints(), refPt(), curPt()
56 { }
57 
69 vpFernClassifier::vpFernClassifier(const std::string &_dataFile, const std::string &_objectName)
70  : vpBasicKeyPoint(), ldetector(), fernClassifier(),
71  gen(0, 256, 5, true, 0.6, 1.5, -CV_PI / 2, CV_PI / 2, -CV_PI / 2, CV_PI / 2), hasLearn(false), threshold(20),
72  nbView(2000), dist(2), nbClassfier(100), ClassifierSize(11), nbOctave(2), patchSize(32), radius(7), nbPoints(200),
73  blurImage(true), radiusBlur(7), sigmaBlur(1), nbMinPoint(10),
74  curImg(), objKeypoints(), modelROI_Ref(), modelROI(), modelPoints(), imgKeypoints(), refPt(), curPt()
75 {
76  this->load(_dataFile, _objectName);
77 }
78 
83 vpFernClassifier::~vpFernClassifier()
84 { }
85 
92 void vpFernClassifier::init()
93 {
94  hasLearn = false;
95  nbClassfier = 100;
96  ClassifierSize = 11;
97  nbPoints = 200;
98  blurImage = true;
99  radiusBlur = 7;
100  sigmaBlur = 1;
101  patchSize = 32;
102  radius = 7;
103  threshold = 20;
104  nbOctave = 2;
105  nbView = 2000;
106  dist = 2;
107  nbMinPoint = 10;
108 }
109 
113 void vpFernClassifier::train()
114 {
115  // initialise detector
116  cv::LDetector d(radius, threshold, nbOctave, nbView, patchSize, dist);
117 
118  // blur
119  cv::Mat obj = (cv::Mat)curImg;
120 
121  if (this->getBlurSetting()) {
122  cv::GaussianBlur(obj, obj, cv::Size(getBlurSize(), getBlurSize()), getBlurSigma(), getBlurSigma());
123  }
124 
125  // build pyramid
126  std::vector<cv::Mat> objpyr;
127  cv::buildPyramid(obj, objpyr, d.nOctaves - 1);
128 
129  // getPoints
130  d.getMostStable2D(obj, objKeypoints, 100, gen);
131 
132  ldetector = d;
133 
134  // train classifier
135  modelROI = cv::Rect(0, 0, objpyr[0].cols, objpyr[0].rows);
136  ldetector.getMostStable2D(objpyr[0], modelPoints, 100, gen);
137 
138  fernClassifier.trainFromSingleView(objpyr[0], modelPoints, patchSize, (int)modelPoints.size(), 100, 11, 10000,
139  cv::FernClassifier::COMPRESSION_NONE, gen);
140 
141  /* from OpenCV format to ViSP format */
142  referenceImagePointsList.resize(0);
143  for (unsigned int i = 0; i < modelPoints.size(); i += 1) {
144  vpImagePoint ip(modelPoints[i].pt.y + modelROI_Ref.y, modelPoints[i].pt.x + modelROI_Ref.x);
145  referenceImagePointsList.push_back(ip);
146  }
147 
148  // set flag
149  hasLearn = true;
150 }
151 
165 unsigned int vpFernClassifier::buildReference(const vpImage<unsigned char> &_I)
166 {
167  this->setImage(_I);
168 
169  train();
170 
171  _reference_computed = true;
172  return (unsigned int)objKeypoints.size();
173 }
174 
193 unsigned int vpFernClassifier::buildReference(const vpImage<unsigned char> &_I, const vpImagePoint &_iP,
194  const unsigned int _height, const unsigned int _width)
195 {
196  if ((_iP.get_i() + _height) >= _I.getHeight() || (_iP.get_j() + _width) >= _I.getWidth()) {
197  vpTRACE("Bad size for the subimage");
198  throw(vpException(vpImageException::notInTheImage, "Bad size for the subimage"));
199  }
200 
201  vpImage<unsigned char> subImage;
202  vpImageTools::crop(_I, (unsigned int)_iP.get_i(), (unsigned int)_iP.get_j(), _height, _width, subImage);
203  this->setImage(subImage);
204 
205  /* initialise a structure containing the region of interest used in the
206  reference image */
207  modelROI_Ref.x = (int)_iP.get_u();
208  modelROI_Ref.y = (int)_iP.get_v();
209  modelROI_Ref.width = (int)_width;
210  modelROI_Ref.height = (int)_height;
211 
212  train();
213 
214  return (unsigned int)objKeypoints.size();
215 }
216 
233 unsigned int vpFernClassifier::buildReference(const vpImage<unsigned char> &_I, const vpRect &_rectangle)
234 {
235  vpImagePoint iP;
236  iP.set_i(_rectangle.getTop());
237  iP.set_j(_rectangle.getLeft());
238  return (this->buildReference(_I, iP, (unsigned int)_rectangle.getHeight(), (unsigned int)_rectangle.getWidth()));
239 }
240 
253 unsigned int vpFernClassifier::matchPoint(const vpImage<unsigned char> &_I)
254 {
255  if (!hasLearn) {
256  vpERROR_TRACE("The object has not been learned. ");
257  throw vpException(vpException::notInitialized, "object is not learned, load database or build the reference ");
258  }
259 
260  setImage(_I);
261  // resize image
262  // cv::resize(_image, image, Size(), 1./imgscale, 1./imgscale,
263  // INTER_CUBIC);
264  cv::Mat img = this->curImg;
265 
266  if (this->getBlurSetting()) {
267  cv::GaussianBlur(img, img, cv::Size(this->getBlurSize(), this->getBlurSize()), this->getBlurSigma(),
268  this->getBlurSigma());
269  }
270 
271  std::vector<cv::Mat> imgPyr;
272  cv::buildPyramid(img, imgPyr, ldetector.nOctaves - 1);
273 
274  ldetector(imgPyr, imgKeypoints, 500);
275 
276  unsigned int m = (unsigned int)modelPoints.size();
277  unsigned int n = (unsigned int)imgKeypoints.size();
278  std::vector<int> bestMatches(m, -1);
279  std::vector<float> maxLogProb(m, -FLT_MAX);
280  std::vector<float> signature;
281  unsigned int totalMatch = 0;
282 
283  /* part of code from OpenCV planarObjectDetector */
284  currentImagePointsList.resize(0);
285  matchedReferencePoints.resize(0);
286 
287  for (unsigned int i = 0; i < n; i++) {
288  cv::KeyPoint kpt = imgKeypoints[i];
289  kpt.pt.x /= (float)(1 << kpt.octave);
290  kpt.pt.y /= (float)(1 << kpt.octave);
291  int k = fernClassifier(imgPyr[(unsigned int)kpt.octave], kpt.pt, signature);
292  if (k >= 0 && (bestMatches[(unsigned int)k] < 0 || signature[(unsigned int)k] > maxLogProb[(unsigned int)k])) {
293  maxLogProb[(unsigned int)k] = signature[(unsigned int)k];
294  bestMatches[(unsigned int)k] = (int)i;
295  totalMatch++;
296 
297  vpImagePoint ip_cur(imgKeypoints[i].pt.y, imgKeypoints[i].pt.x);
298 
299  currentImagePointsList.push_back(ip_cur);
300  matchedReferencePoints.push_back((unsigned int)k);
301  }
302  }
303 
304  refPt.resize(0);
305  curPt.resize(0);
306  for (unsigned int i = 0; i < m; i++) {
307  if (bestMatches[i] >= 0) {
308  refPt.push_back(modelPoints[i].pt);
309  curPt.push_back(imgKeypoints[(unsigned int)bestMatches[i]].pt);
310  }
311  }
312 
313  return totalMatch;
314 }
315 
329 unsigned int vpFernClassifier::matchPoint(const vpImage<unsigned char> &_I, const vpImagePoint &_iP,
330  const unsigned int _height, const unsigned int _width)
331 {
332  if ((_iP.get_i() + _height) >= _I.getHeight() || (_iP.get_j() + _width) >= _I.getWidth()) {
333  vpTRACE("Bad size for the subimage");
334  throw(vpException(vpImageException::notInTheImage, "Bad size for the subimage"));
335  }
336 
337  vpImage<unsigned char> subImage;
338 
339  vpImageTools::crop(_I, (unsigned int)_iP.get_i(), (unsigned int)_iP.get_j(), _height, _width, subImage);
340 
341  return this->matchPoint(subImage);
342 }
343 
355 unsigned int vpFernClassifier::matchPoint(const vpImage<unsigned char> &_I, const vpRect &_rectangle)
356 {
357  vpImagePoint iP;
358  iP.set_i(_rectangle.getTop());
359  iP.set_j(_rectangle.getLeft());
360  return (this->matchPoint(_I, iP, (unsigned int)_rectangle.getHeight(), (unsigned int)_rectangle.getWidth()));
361 }
362 
380 void vpFernClassifier::display(const vpImage<unsigned char> &_Iref, const vpImage<unsigned char> &_Icurrent,
381  unsigned int size)
382 {
383  for (unsigned int i = 0; i < matchedReferencePoints.size(); i++) {
384  vpDisplay::displayCross(_Iref, referenceImagePointsList[matchedReferencePoints[i]], size, vpColor::red);
385  vpDisplay::displayCross(_Icurrent, currentImagePointsList[i], size, vpColor::green);
386  }
387 }
388 
400 void vpFernClassifier::display(const vpImage<unsigned char> &_Icurrent, unsigned int size, const vpColor &color)
401 {
402  for (unsigned int i = 0; i < matchedReferencePoints.size(); i++) {
403  vpDisplay::displayCross(_Icurrent, currentImagePointsList[i], size, color);
404  }
405 }
406 
407 /* IO METHODS */
408 
418 void vpFernClassifier::load(const std::string &_dataFile, const std::string & /*_objectName*/)
419 {
420  std::cout << " > Load data for the planar object detector..." << std::endl;
421 
422  /* part of code from OpenCV planarObjectDetector */
423  cv::FileStorage fs(_dataFile, cv::FileStorage::READ);
424  cv::FileNode node = fs.getFirstTopLevelNode();
425 
426  cv::FileNodeIterator it = node["model-roi"].begin(), it_end;
427  it >> modelROI.x >> modelROI.y >> modelROI.width >> modelROI.height;
428 
429  ldetector.read(node["detector"]);
430  fernClassifier.read(node["fern-classifier"]);
431 
432  const cv::FileNode node_ = node["model-points"];
433  cv::read(node_, modelPoints);
434 
435  cv::LDetector d(radius, threshold, nbOctave, nbView, patchSize, dist);
436  ldetector = d;
437  hasLearn = true;
438 }
439 
447 void vpFernClassifier::record(const std::string &_objectName, const std::string &_dataFile)
448 {
449  /* part of code from OpenCV planarObjectDetector */
450  cv::FileStorage fs(_dataFile, cv::FileStorage::WRITE);
451 
452  cv::WriteStructContext ws(fs, _objectName, CV_NODE_MAP);
453 
454  {
455  cv::WriteStructContext wsroi(fs, "model-roi", CV_NODE_SEQ + CV_NODE_FLOW);
456  cv::write(fs, modelROI_Ref.x);
457  cv::write(fs, modelROI_Ref.y);
458  cv::write(fs, modelROI_Ref.width);
459  cv::write(fs, modelROI_Ref.height);
460  }
461 
462  ldetector.write(fs, "detector");
463  cv::write(fs, "model-points", modelPoints);
464  fernClassifier.write(fs, "fern-classifier");
465 }
466 
473 void vpFernClassifier::setImage(const vpImage<unsigned char> &I)
474 {
475  vpImageConvert::convert(I, curImg);
476 }
477 
478 #elif !defined(VISP_BUILD_SHARED_LIBS)
479 // Work around to avoid warning: libvisp_vision.a(vpFernClassifier.cpp.o) has
480 // no symbols
481 void dummy_vpFernClassifier() { };
482 #endif
class that defines what is a keypoint. This class provides all the basic elements to implement classe...
Class to define RGB colors available for display functionalities.
Definition: vpColor.h:152
static const vpColor red
Definition: vpColor.h:211
static const vpColor green
Definition: vpColor.h:214
static void displayCross(const vpImage< unsigned char > &I, const vpImagePoint &ip, unsigned int size, const vpColor &color, unsigned int thickness=1)
error that can be emitted by ViSP classes.
Definition: vpException.h:59
@ notInitialized
Used to indicate that a parameter is not initialized.
Definition: vpException.h:86
static void convert(const vpImage< unsigned char > &src, vpImage< vpRGBa > &dest)
@ notInTheImage
Pixel not in the image.
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
Definition: vpImagePoint.h:82
void set_j(double jj)
Definition: vpImagePoint.h:302
double get_j() const
Definition: vpImagePoint.h:125
void set_i(double ii)
Definition: vpImagePoint.h:291
double get_u() const
Definition: vpImagePoint.h:136
double get_i() const
Definition: vpImagePoint.h:114
double get_v() const
Definition: vpImagePoint.h:147
static void crop(const vpImage< Type > &I, double roi_top, double roi_left, unsigned int roi_height, unsigned int roi_width, vpImage< Type > &crop, unsigned int v_scale=1, unsigned int h_scale=1)
Definition: vpImageTools.h:301
unsigned int getWidth() const
Definition: vpImage.h:242
unsigned int getHeight() const
Definition: vpImage.h:184
Defines a rectangle in the plane.
Definition: vpRect.h:76
double getWidth() const
Definition: vpRect.h:224
double getLeft() const
Definition: vpRect.h:170
double getHeight() const
Definition: vpRect.h:163
double getTop() const
Definition: vpRect.h:189
#define vpTRACE
Definition: vpDebug.h:411
#define vpERROR_TRACE
Definition: vpDebug.h:388