Visual Servoing Platform  version 3.6.1 under development (2024-02-13)
vpDetectorDNNOpenCV.h
1 /*
2  * ViSP, open source Visual Servoing Platform software.
3  * Copyright (C) 2005 - 2023 by Inria. All rights reserved.
4  *
5  * This software is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  * See the file LICENSE.txt at the root directory of this source
10  * distribution for additional information about the GNU GPL.
11  *
12  * For using ViSP with software that can not be combined with the GNU
13  * GPL, please contact Inria about acquiring a ViSP Professional
14  * Edition License.
15  *
16  * See https://visp.inria.fr for more information.
17  *
18  * This software was developed at:
19  * Inria Rennes - Bretagne Atlantique
20  * Campus Universitaire de Beaulieu
21  * 35042 Rennes Cedex
22  * France
23  *
24  * If you have questions regarding the use of this file, please contact
25  * Inria at visp@inria.fr
26  *
27  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
28  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
29  *
30  * Description:
31  * DNN object detection using OpenCV DNN module.
32  */
33 #ifndef _vpDetectorDNN_h_
34 #define _vpDetectorDNN_h_
35 
36 #include <visp3/core/vpConfig.h>
37 
38 // Check if std:c++17 or higher.
39 // Here we cannot use (VISP_CXX_STANDARD >= VISP_CXX_STANDARD_17) in the declaration of the class
40 #if (VISP_HAVE_OPENCV_VERSION >= 0x030403) && defined(HAVE_OPENCV_DNN) && \
41  ((__cplusplus >= 201703L) || (defined(_MSVC_LANG) && (_MSVC_LANG >= 201703L)))
42 
43 #include <map>
44 #include <string>
45 #include <vector>
46 
47 #include <opencv2/dnn.hpp>
48 
49 #include <visp3/core/vpColor.h>
50 #include <visp3/core/vpDisplay.h>
51 #include <visp3/core/vpImage.h>
52 #include <visp3/core/vpRect.h>
53 
54 #include <optional>
55 #ifdef VISP_HAVE_NLOHMANN_JSON
56 #include <nlohmann/json.hpp>
57 using json = nlohmann::json;
58 #endif
59 
82 class VISP_EXPORT vpDetectorDNNOpenCV
83 {
84 public:
90  typedef enum DNNResultsParsingType
91  {
92  USER_SPECIFIED = 0,
93  FASTER_RCNN = 1,
94  SSD_MOBILENET = 2,
95  RESNET_10 = 3,
96  YOLO_V3 = 4,
97  YOLO_V4 = 5,
98  YOLO_V5 = 6,
99  YOLO_V7 = 7,
100  YOLO_V8 = 8,
101  COUNT = 9
102  } DNNResultsParsingType;
103 
104  typedef struct DetectionCandidates
105  {
106  std::vector< float > m_confidences;
107  std::vector< cv::Rect > m_boxes;
108  std::vector< int > m_classIds;
109  } DetectionCandidates;
110 
116  typedef class DetectedFeatures2D
117  {
118  protected:
119  vpRect m_bbox;
120  double m_score;
121  unsigned int m_cls;
122  std::optional<std::string> m_classname;
123  public:
135  inline explicit DetectedFeatures2D(double u_min, double u_max
136  , double v_min, double v_max
137  , unsigned int cls, double score
138  , const std::optional<std::string> &classname
139  )
140  : m_bbox(vpImagePoint(v_min, u_min), vpImagePoint(v_max, u_max))
141  , m_score(score)
142  , m_cls(cls)
143  {
144  if (classname) {
145  m_classname = classname;
146  }
147  else {
148  m_classname = std::nullopt;
149  }
150  };
151 
155  inline vpRect getBoundingBox() const { return m_bbox; }
159  inline double getConfidenceScore() const { return m_score; }
163  inline unsigned int getClassId() const { return m_cls; }
167  inline std::optional<std::string> getClassName() const { return m_classname; }
168 
169  template < typename Type >
170  void display(const vpImage< Type > &img, const vpColor &color = vpColor::blue, unsigned int thickness = 1) const;
171 
172  friend vpDetectorDNNOpenCV;
173  } DetectedFeatures2D;
174 
179  typedef class NetConfig
180  {
181  private:
182  float m_confThreshold;
183  float m_nmsThreshold;
184  std::vector<std::string> m_classNames;
185  cv::Size m_inputSize;
186  double m_filterSizeRatio;
188  cv::Scalar m_mean;
189  double m_scaleFactor;
190  bool m_swapRB; /*<! If true, swap R and B for mean subtraction, e.g. when a model has been trained on BGR image format.*/
191  DNNResultsParsingType m_parsingMethodType;
192  std::string m_modelFilename;
193  std::string m_modelConfigFilename; /*<! Path towards the model additional configuration file, e.g. pbtxt file.*/
194  std::string m_framework;
196 #ifdef VISP_HAVE_NLOHMANN_JSON
204  friend inline void from_json(const json &j, NetConfig &config)
205  {
206  config.m_confThreshold = j.value("confidenceThreshold", config.m_confThreshold);
207  if (config.m_confThreshold <= 0) {
208  throw vpException(vpException::badValue, "Confidence threshold should be > 0");
209  }
210 
211  config.m_nmsThreshold = j.value("nmsThreshold", config.m_nmsThreshold);
212  if (config.m_nmsThreshold <= 0) {
213  throw vpException(vpException::badValue, "Confidence threshold should be > 0");
214  }
215 
216  config.m_filterSizeRatio = j.value("filterSizeRatio", config.m_filterSizeRatio);
217 
218  config.m_classNames = j.value("classNames", config.m_classNames);
219 
220  std::pair<unsigned int, unsigned int> resolution = j.value("resolution", std::pair<unsigned int, unsigned int>(config.m_inputSize.width, config.m_inputSize.height));
221  config.m_inputSize.width = resolution.first;
222  config.m_inputSize.height = resolution.second;
223 
224  std::vector<double> v_mean = j.value("mean", std::vector<double>({ config.m_mean[0], config.m_mean[1], config.m_mean[2] }));
225  if (v_mean.size() != 3) {
226  throw(vpException(vpException::dimensionError, "Mean should have size = 3"));
227  }
228  config.m_mean = cv::Scalar(v_mean[0], v_mean[1], v_mean[2]);
229 
230  config.m_scaleFactor = j.value("scale", config.m_scaleFactor);
231  config.m_swapRB = j.value("swapRB", config.m_swapRB);
232  config.m_parsingMethodType = dnnResultsParsingTypeFromString(j.value("parsingType", dnnResultsParsingTypeToString(config.m_parsingMethodType)));
233  config.m_modelFilename = j.value("modelFile", config.m_modelFilename);
234  config.m_modelConfigFilename = j.value("configurationFile", config.m_modelConfigFilename);
235  config.m_framework = j.value("framework", config.m_framework);
236  }
237 
244  friend inline void to_json(json &j, const NetConfig &config)
245  {
246  std::pair<unsigned int, unsigned int> resolution = { config.m_inputSize.width, config.m_inputSize.height };
247  std::vector<double> v_mean = { config.m_mean[0], config.m_mean[1], config.m_mean[2] };
248  j = json {
249  {"confidenceThreshold", config.m_confThreshold } ,
250  {"nmsThreshold" , config.m_nmsThreshold } ,
251  {"filterSizeRatio" , config.m_filterSizeRatio} ,
252  {"classNames" , config.m_classNames } ,
253  {"resolution" , resolution } ,
254  {"mean" , v_mean } ,
255  {"scale" , config.m_scaleFactor } ,
256  {"swapRB" , config.m_swapRB } ,
257  {"parsingType" , dnnResultsParsingTypeToString(config.m_parsingMethodType) },
258  {"modelFile" , config.m_modelFilename } ,
259  {"configurationFile" , config.m_modelConfigFilename } ,
260  {"framework" , config.m_framework }
261  };
262  }
263 #endif
264 
265  public:
288  inline static std::vector<std::string> parseClassNamesFile(const std::string &filename)
289  {
290  std::vector<std::string> classNames;
291  std::ifstream ifs(filename);
292  std::string line;
293  while (getline(ifs, line)) {
294  if (line.find("[") == std::string::npos) {
295  classNames.push_back(line);
296  }
297  else {
298  std::string lineWithoutBracket;
299  if (line.find("[") != std::string::npos) {
300  lineWithoutBracket = line.substr(line.find("[") + 1, line.size() - 2); // Remove opening and closing brackets
301  }
302 
303  while (!lineWithoutBracket.empty()) {
304  std::string className;
305  auto start_pos = lineWithoutBracket.find("\"");
306  auto end_pos = lineWithoutBracket.find("\"", start_pos + 1);
307  className = lineWithoutBracket.substr(start_pos + 1, end_pos - (start_pos + 1));
308  if (!className.empty()) {
309  classNames.push_back(className);
310  lineWithoutBracket = lineWithoutBracket.substr(end_pos + 1);
311  }
312  }
313  }
314  }
315  return classNames;
316  }
317 
321  inline NetConfig()
322  : m_confThreshold(0.5f)
323  , m_nmsThreshold(0.4f)
324  , m_classNames()
325  , m_inputSize(300, 300)
326  , m_filterSizeRatio(0.)
327  , m_mean(127.5, 127.5, 127.5)
328  , m_scaleFactor(2.0 / 255.0)
329  , m_swapRB(true)
330  , m_parsingMethodType(vpDetectorDNNOpenCV::USER_SPECIFIED)
331  , m_modelFilename()
332  , m_modelConfigFilename()
333  , m_framework()
334  {
335 
336  }
337 
338  inline NetConfig(const NetConfig &config)
339  : m_confThreshold(config.m_confThreshold)
340  , m_nmsThreshold(config.m_nmsThreshold)
341  , m_classNames(config.m_classNames)
342  , m_inputSize(config.m_inputSize.width, config.m_inputSize.height)
343  , m_filterSizeRatio(config.m_filterSizeRatio)
344  , m_mean(cv::Scalar(config.m_mean[0], config.m_mean[1], config.m_mean[2]))
345  , m_scaleFactor(config.m_scaleFactor)
346  , m_swapRB(config.m_swapRB)
347  , m_parsingMethodType(config.m_parsingMethodType)
348  , m_modelFilename(config.m_modelFilename)
349  , m_modelConfigFilename(config.m_modelConfigFilename)
350  , m_framework(config.m_framework)
351  {
352 
353  }
354 
372  inline NetConfig(float confThresh, const float &nmsThresh, const std::vector<std::string> &classNames, const cv::Size &dnnInputSize, const double &filterSizeRatio = 0.
373  , const cv::Scalar &mean = cv::Scalar(127.5, 127.5, 127.5), const double &scaleFactor = 2. / 255., const bool &swapRB = true
374  , const DNNResultsParsingType &parsingType = vpDetectorDNNOpenCV::USER_SPECIFIED, const std::string &modelFilename = "", const std::string &configFilename = "", const std::string &framework = "")
375  : m_confThreshold(confThresh)
376  , m_nmsThreshold(nmsThresh)
377  , m_classNames(classNames)
378  , m_inputSize(dnnInputSize)
379  , m_filterSizeRatio(filterSizeRatio)
380  , m_mean(mean)
381  , m_scaleFactor(scaleFactor)
382  , m_swapRB(swapRB)
383  , m_parsingMethodType(parsingType)
384  , m_modelFilename(modelFilename)
385  , m_modelConfigFilename(configFilename)
386  , m_framework(framework)
387  { }
388 
406  inline NetConfig(const float &confThresh, const float &nmsThresh, const std::string &classNamesFile, const cv::Size &dnnInputSize, const double &filterSizeRatio = 0.
407  , const cv::Scalar &mean = cv::Scalar(127.5, 127.5, 127.5), const double &scaleFactor = 2. / 255., const bool &swapRB = true
408  , const DNNResultsParsingType &parsingType = vpDetectorDNNOpenCV::USER_SPECIFIED, const std::string &modelFilename = "", const std::string &configFilename = "", const std::string &framework = "")
409  : m_confThreshold(confThresh)
410  , m_nmsThreshold(nmsThresh)
411  , m_inputSize(dnnInputSize)
412  , m_filterSizeRatio(filterSizeRatio)
413  , m_mean(mean)
414  , m_scaleFactor(scaleFactor)
415  , m_swapRB(swapRB)
416  , m_parsingMethodType(parsingType)
417  , m_modelFilename(modelFilename)
418  , m_modelConfigFilename(configFilename)
419  , m_framework(framework)
420  {
421  m_classNames = parseClassNamesFile(classNamesFile);
422  }
423 
424  inline std::string toString() const
425  {
426  std::string text;
427  text += "Model : " + m_modelFilename + "\n";
428  text += "Type : " + vpDetectorDNNOpenCV::dnnResultsParsingTypeToString(m_parsingMethodType) + "\n";
429  text += "Config (optional): " + (m_modelConfigFilename.empty() ? "\"None\"" : m_modelConfigFilename) + "\n";
430  text += "Framework (optional): " + (m_framework.empty() ? "\"None\"" : m_framework) + "\n";
431  text += "Width x Height : " + std::to_string(m_inputSize.width) + " x " + std::to_string(m_inputSize.height) + "\n";
432  text += "Mean RGB : " + std::to_string(m_mean[0]) + " " + std::to_string(m_mean[1]) + " " + std::to_string(m_mean[2]) + "\n";
433  text += "Scale : " + std::to_string(m_scaleFactor) + "\n";
434  text += "Swap RB? : " + (m_swapRB ? std::string("true") : std::string("false")) + "\n";
435  text += "Confidence threshold : " + std::to_string(m_confThreshold) + "\n";
436  text += "NMS threshold : " + std::to_string(m_nmsThreshold) + "\n";
437  text += "Filter threshold : " +
438  (m_filterSizeRatio > std::numeric_limits<double>::epsilon() ? std::to_string(m_filterSizeRatio)
439  : "disabled") + "\n";
440  return text;
441  }
442 
443  friend inline std::ostream &operator<<(std::ostream &os, const NetConfig &config)
444  {
445  os << config.toString();
446  return os;
447  }
448 
449  NetConfig &operator=(const NetConfig &config)
450  {
451  m_confThreshold = config.m_confThreshold;
452  m_nmsThreshold = config.m_nmsThreshold;
453  m_classNames = config.m_classNames;
454  m_inputSize = cv::Size(config.m_inputSize.width, config.m_inputSize.height);
455  m_filterSizeRatio = config.m_filterSizeRatio;
456  m_mean = cv::Scalar(config.m_mean[0], config.m_mean[1], config.m_mean[2]);
457  m_scaleFactor = config.m_scaleFactor;
458  m_swapRB = config.m_swapRB;
459  m_parsingMethodType = config.m_parsingMethodType;
460  m_modelFilename = config.m_modelFilename;
461  m_modelConfigFilename = config.m_modelConfigFilename;
462  m_framework = config.m_framework;
463  return *this;
464  }
465 
466  friend vpDetectorDNNOpenCV;
467  } NetConfig;
468 
469  static std::string getAvailableDnnResultsParsingTypes();
470  static std::string dnnResultsParsingTypeToString(const DNNResultsParsingType &type);
471  static DNNResultsParsingType dnnResultsParsingTypeFromString(const std::string &name);
472  static std::vector<std::string> parseClassNamesFile(const std::string &filename);
473  vpDetectorDNNOpenCV();
474  vpDetectorDNNOpenCV(const NetConfig &config, const DNNResultsParsingType &typeParsingMethod, void (*parsingMethod)(DetectionCandidates &, std::vector<cv::Mat> &, const NetConfig &) = postProcess_unimplemented);
475 #ifdef VISP_HAVE_NLOHMANN_JSON
476  vpDetectorDNNOpenCV(const std::string &jsonPath, void (*parsingMethod)(DetectionCandidates &, std::vector<cv::Mat> &, const NetConfig &) = postProcess_unimplemented);
477  void initFromJSON(const std::string &jsonPath);
478  void saveConfigurationInJSON(const std::string &jsonPath) const;
479 #endif
480  virtual ~vpDetectorDNNOpenCV();
481 
482  virtual bool detect(const vpImage<unsigned char> &I, std::vector<DetectedFeatures2D> &output);
483  virtual bool detect(const vpImage<unsigned char> &I, std::map< std::string, std::vector<DetectedFeatures2D>> &output);
484  virtual bool detect(const vpImage<unsigned char> &I, std::vector< std::pair<std::string, std::vector<DetectedFeatures2D>>> &output);
485  virtual bool detect(const vpImage<vpRGBa> &I, std::vector<DetectedFeatures2D> &output);
486  virtual bool detect(const vpImage<vpRGBa> &I, std::map< std::string, std::vector<DetectedFeatures2D>> &output);
487  virtual bool detect(const vpImage<vpRGBa> &I, std::vector< std::pair<std::string, std::vector<DetectedFeatures2D>>> &output);
488  virtual bool detect(const cv::Mat &I, std::vector<DetectedFeatures2D> &output);
489  virtual bool detect(const cv::Mat &I, std::map< std::string, std::vector<DetectedFeatures2D>> &output);
490  virtual bool detect(const cv::Mat &I, std::vector< std::pair<std::string, std::vector<DetectedFeatures2D>>> &output);
491 
492  void readNet(const std::string &model, const std::string &config = "", const std::string &framework = "");
493 
494  void setNetConfig(const NetConfig &config);
495  void setConfidenceThreshold(const float &confThreshold);
496  void setNMSThreshold(const float &nmsThreshold);
497  void setDetectionFilterSizeRatio(const double &sizeRatio);
498  void setInputSize(const int &width, const int &height);
499  void setMean(const double &meanR, const double &meanG, const double &meanB);
500  void setPreferableBackend(const int &backendId);
501  void setPreferableTarget(const int &targetId);
502  void setScaleFactor(const double &scaleFactor);
503  void setSwapRB(const bool &swapRB);
504  void setParsingMethod(const DNNResultsParsingType &typeParsingMethod, void (*parsingMethod)(DetectionCandidates &, std::vector<cv::Mat> &, const NetConfig &) = postProcess_unimplemented);
505  inline const NetConfig &getNetConfig() const
506  {
507  return m_netConfig;
508  }
509 
510 #ifdef VISP_HAVE_NLOHMANN_JSON
518  friend inline void from_json(const json &j, vpDetectorDNNOpenCV &network)
519  {
520  network.m_netConfig = j.value("networkSettings", network.m_netConfig);
521  }
522 
529  friend inline void to_json(json &j, const vpDetectorDNNOpenCV &network)
530  {
531  j = json {
532  {"networkSettings", network.m_netConfig}
533  };
534  }
535 #endif
536 
537  friend inline std::ostream &operator<<(std::ostream &os, const vpDetectorDNNOpenCV &network)
538  {
539  os << network.m_netConfig;
540  return os;
541  }
542 
543 protected:
544 #if (VISP_HAVE_OPENCV_VERSION == 0x030403)
545  std::vector<cv::String> getOutputsNames();
546 #endif
547  std::vector<DetectedFeatures2D>
548  filterDetectionSingleClassInput(const std::vector<DetectedFeatures2D> &detected_features, const double minRatioOfAreaOk);
549 
550  std::vector<DetectedFeatures2D>
551  filterDetectionMultiClassInput(const std::vector<DetectedFeatures2D> &detected_features, const double minRatioOfAreaOk);
552 
553  std::map<std::string, std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D>>
554  filterDetectionMultiClassInput(const std::map< std::string, std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D>> &detected_features, const double minRatioOfAreaOk);
555 
556  void postProcess(DetectionCandidates &proposals);
557 
558  void postProcess_YoloV3_V4(DetectionCandidates &proposals, std::vector<cv::Mat> &dnnRes, const NetConfig &netConfig);
559 
560  void postProcess_YoloV5_V7(DetectionCandidates &proposals, std::vector<cv::Mat> &dnnRes, const NetConfig &netConfig);
561 
562  void postProcess_YoloV8(DetectionCandidates &proposals, std::vector<cv::Mat> &dnnRes, const NetConfig &netConfig);
563 
564  void postProcess_FasterRCNN(DetectionCandidates &proposals, std::vector<cv::Mat> &dnnRes, const NetConfig &netConfig);
565 
566 #if defined(VISP_BUILD_DEPRECATED_FUNCTIONS)
567  void postProcess_SSD_MobileNet(DetectionCandidates &proposals, std::vector<cv::Mat> &dnnRes, const NetConfig &netConfig);
568 #endif
569 
570  void postProcess_ResNet_10(DetectionCandidates &proposals, std::vector<cv::Mat> &dnnRes, const NetConfig &netConfig);
571 
572  static void postProcess_unimplemented(DetectionCandidates &proposals, std::vector<cv::Mat> &dnnRes, const NetConfig &netConfig);
573 
575  bool m_applySizeFilterAfterNMS;
577  cv::Mat m_blob;
579  vpImage<vpRGBa> m_I_color;
581  cv::Mat m_img;
583  std::vector<int> m_indices;
585  cv::dnn::Net m_net;
587  NetConfig m_netConfig;
589  std::vector<cv::String> m_outNames;
591  std::vector<cv::Mat> m_dnnRes;
593  void (*m_parsingMethod)(DetectionCandidates &, std::vector<cv::Mat> &, const NetConfig &);
594 };
595 
603 template < typename Type >
604 inline void
605 vpDetectorDNNOpenCV::DetectedFeatures2D::display(const vpImage< Type > &img, const vpColor &color, unsigned int thickness) const
606 {
607  vpDisplay::displayRectangle(img, m_bbox, color, false, thickness);
608 
609  std::stringstream ss;
610  if (m_classname) {
611  ss << *m_classname;
612  }
613  else {
614  ss << m_cls;
615  }
616  ss << "(" << std::setprecision(4) << m_score * 100. << "%)";
617  vpDisplay::displayText(img, m_bbox.getTopRight(), ss.str(), color);
618 }
619 #endif
620 #endif
Class to define RGB colors available for display functionalities.
Definition: vpColor.h:152
static const vpColor blue
Definition: vpColor.h:217
static void displayRectangle(const vpImage< unsigned char > &I, const vpImagePoint &topLeft, unsigned int width, unsigned int height, const vpColor &color, bool fill=false, unsigned int thickness=1)
static void displayText(const vpImage< unsigned char > &I, const vpImagePoint &ip, const std::string &s, const vpColor &color)
error that can be emitted by ViSP classes.
Definition: vpException.h:59
@ badValue
Used to indicate that a value is not in the allowed range.
Definition: vpException.h:85
@ dimensionError
Bad dimension.
Definition: vpException.h:83
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
Definition: vpImagePoint.h:82
Definition of the vpImage class member functions.
Definition: vpImage.h:69
Defines a rectangle in the plane.
Definition: vpRect.h:76
vpImagePoint getTopRight() const
Definition: vpRect.h:209
void display(vpImage< unsigned char > &I, const std::string &title)
Display a gray-scale image.