Visual Servoing Platform  version 3.6.1 under development (2023-11-16)
vpDetectorDNNOpenCV Class Reference

#include <visp3/detection/vpDetectorDNNOpenCV.h>

Classes

class  DetectedFeatures2D
 
struct  DetectionCandidates
 
struct  NetConfig
 

Public Types

enum  DNNResultsParsingType {
  USER_SPECIFIED = 0 , FASTER_RCNN = 1 , SSD_MOBILENET = 2 , RESNET_10 = 3 ,
  YOLO_V3 = 4 , YOLO_V4 = 5 , YOLO_V5 = 6 , YOLO_V7 = 7 ,
  YOLO_V8 = 8 , COUNT = 9
}
 
typedef enum vpDetectorDNNOpenCV::DNNResultsParsingType DNNResultsParsingType
 
typedef struct vpDetectorDNNOpenCV::DetectionCandidates DetectionCandidates
 
typedef class vpDetectorDNNOpenCV::DetectedFeatures2D DetectedFeatures2D
 
typedef class vpDetectorDNNOpenCV::NetConfig NetConfig
 

Public Member Functions

 vpDetectorDNNOpenCV ()
 
 vpDetectorDNNOpenCV (const NetConfig &config, const DNNResultsParsingType &typeParsingMethod, void(*parsingMethod)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)=postProcess_unimplemented)
 
 vpDetectorDNNOpenCV (const std::string &jsonPath, void(*parsingMethod)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)=postProcess_unimplemented)
 
void initFromJSON (const std::string &jsonPath)
 
void saveConfigurationInJSON (const std::string &jsonPath) const
 
virtual ~vpDetectorDNNOpenCV ()
 
virtual bool detect (const vpImage< unsigned char > &I, std::vector< DetectedFeatures2D > &output)
 
virtual bool detect (const vpImage< unsigned char > &I, std::map< std::string, std::vector< DetectedFeatures2D >> &output)
 
virtual bool detect (const vpImage< unsigned char > &I, std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &output)
 
virtual bool detect (const vpImage< vpRGBa > &I, std::vector< DetectedFeatures2D > &output)
 
virtual bool detect (const vpImage< vpRGBa > &I, std::map< std::string, std::vector< DetectedFeatures2D >> &output)
 
virtual bool detect (const vpImage< vpRGBa > &I, std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &output)
 
virtual bool detect (const cv::Mat &I, std::vector< DetectedFeatures2D > &output)
 
virtual bool detect (const cv::Mat &I, std::map< std::string, std::vector< DetectedFeatures2D >> &output)
 
virtual bool detect (const cv::Mat &I, std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &output)
 
void readNet (const std::string &model, const std::string &config="", const std::string &framework="")
 
void setNetConfig (const NetConfig &config)
 
void setConfidenceThreshold (const float &confThreshold)
 
void setNMSThreshold (const float &nmsThreshold)
 
void setDetectionFilterSizeRatio (const double &sizeRatio)
 
void setInputSize (const int &width, const int &height)
 
void setMean (const double &meanR, const double &meanG, const double &meanB)
 
void setPreferableBackend (const int &backendId)
 
void setPreferableTarget (const int &targetId)
 
void setScaleFactor (const double &scaleFactor)
 
void setSwapRB (const bool &swapRB)
 
void setParsingMethod (const DNNResultsParsingType &typeParsingMethod, void(*parsingMethod)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)=postProcess_unimplemented)
 
const NetConfiggetNetConfig () const
 

Static Public Member Functions

static std::string getAvailableDnnResultsParsingTypes ()
 
static std::string dnnResultsParsingTypeToString (const DNNResultsParsingType &type)
 
static DNNResultsParsingType dnnResultsParsingTypeFromString (const std::string &name)
 
static std::vector< std::string > parseClassNamesFile (const std::string &filename)
 

Protected Member Functions

std::vector< cv::String > getOutputsNames ()
 
std::vector< DetectedFeatures2DfilterDetectionSingleClassInput (const std::vector< DetectedFeatures2D > &detected_features, const double minRatioOfAreaOk)
 
std::vector< DetectedFeatures2DfilterDetectionMultiClassInput (const std::vector< DetectedFeatures2D > &detected_features, const double minRatioOfAreaOk)
 
std::map< std::string, std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D > > filterDetectionMultiClassInput (const std::map< std::string, std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D >> &detected_features, const double minRatioOfAreaOk)
 
void postProcess (DetectionCandidates &proposals)
 
void postProcess_YoloV3_V4 (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
 
void postProcess_YoloV5_V7 (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
 
void postProcess_YoloV8 (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
 
void postProcess_FasterRCNN (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
 
void postProcess_SSD_MobileNet (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
 
void postProcess_ResNet_10 (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
 

Static Protected Member Functions

static void postProcess_unimplemented (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
 

Protected Attributes

bool m_applySizeFilterAfterNMS
 
cv::Mat m_blob
 
vpImage< vpRGBam_I_color
 
cv::Mat m_img
 
std::vector< int > m_indices
 
cv::dnn::Net m_net
 
NetConfig m_netConfig
 
std::vector< cv::String > m_outNames
 
std::vector< cv::Mat > m_dnnRes
 
void(* m_parsingMethod )(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)
 

Friends

void from_json (const json &j, vpDetectorDNNOpenCV &network)
 
void to_json (json &j, const vpDetectorDNNOpenCV &network)
 
std::ostream & operator<< (std::ostream &os, const vpDetectorDNNOpenCV &network)
 

Detailed Description

json namespace shortcut

This class is a wrapper over the OpenCV DNN module and specialized to handle object detection task.

This class supports the following networks dedicated to object detection:

  • Faster-RCNN, see usage to detect objects belonging to the COCO dataset using Faster-RCNN network
  • SSD MobileNet, see usage to detect objects belonging to the COCO dataset using MobileNet SSD network
  • ResNet 10, see usage for Face detection
  • Yolo v3, see usage to detect objects belonging to the COCO dataset using Yolo v3 network
  • Yolo v4, see usage to detect objects belonging to the COCO dataset using Yolo v4 network
  • Yolo v5, see usage to detect objects belonging to the COCO dataset using Yolo v5 network
  • Yolo v7, see usage to detect objects belonging to the COCO dataset using Yolo v7 network
  • Yolo v8, see usage to detect objects belonging to the COCO dataset using Yolo v8 network

This class can be initialized from a JSON file if ViSP has been compiled with NLOHMANN JSON (see JSON for modern C++ to see how to do it). Examples of such JSON files can be found in the tutorial folder.

See also
Tutorial: Deep learning object detection
Examples
tutorial-dnn-object-detection-live.cpp, and tutorial-megapose-live-single-object-tracking.cpp.

Definition at line 78 of file vpDetectorDNNOpenCV.h.

Member Typedef Documentation

◆ DetectedFeatures2D

◆ DetectionCandidates

◆ DNNResultsParsingType

◆ NetConfig

Member Enumeration Documentation

◆ DNNResultsParsingType

Enumeration listing the types of DNN for which the vpDetectorDNNOpenCV furnishes the methods permitting to parse the raw detection data into vpDetectorDNNOpenCV::DetectedFeatures2D .

Enumerator
USER_SPECIFIED 

The user will give a pointer towards the parsing method to use to parse the raw data resulting from the detection step.

FASTER_RCNN 

The vpDetectorDNNOpenCV object will use the parsing method corresponding to a Faster-RCNN DNN. See vpDetectorDNNOpenCV::postProcess_FasterRCNN for more information.

SSD_MOBILENET 

The vpDetectorDNNOpenCV object will use the parsing method corresponding to a SSD MobileNet DNN. See vpDetectorDNNOpenCV::postProcess_SSD_MobileNet for more information.

RESNET_10 

The vpDetectorDNNOpenCV object will use the parsing method corresponding to a ResNet 10 DNN. See vpDetectorDNNOpenCV::postProcess_ResNet_10 for more information.

YOLO_V3 

The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV3 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV3_V4 for more information.

YOLO_V4 

The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV4 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV3_V4 for more information.

YOLO_V5 

The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV5 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV5_V7 for more information.

YOLO_V7 

The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV7 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV5_V7 for more information.

YOLO_V8 

The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV8 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV8 for more information.

COUNT 

The number of parsing method that come along with the vpDetectorDNNOpenCV class.

Definition at line 86 of file vpDetectorDNNOpenCV.h.

Constructor & Destructor Documentation

◆ vpDetectorDNNOpenCV() [1/3]

vpDetectorDNNOpenCV::vpDetectorDNNOpenCV ( )

Definition at line 143 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig, and setDetectionFilterSizeRatio().

◆ vpDetectorDNNOpenCV() [2/3]

vpDetectorDNNOpenCV::vpDetectorDNNOpenCV ( const NetConfig config,
const DNNResultsParsingType typeParsingMethod,
void(*)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)  parsingMethod = postProcess_unimplemented 
)

Construct a new vpDetectorDNNOpenCV object and, if the model file has been given to the config object, read the net by calling vpDetectorDNNOpenCV::readNet.

Parameters
configThe network configuration.
typeParsingMethodThe type of parsing method that must be used to parse the raw results of the DNN detection step.
parsingMethodIf typeParsingMethod is set to vpDetectorDNNOpenCV::DNNResultsParsingType::USER_SPECIFIED, the parsing method that must be used to parse the raw results of the DNN detection step.

Definition at line 158 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig, readNet(), setDetectionFilterSizeRatio(), and setParsingMethod().

◆ vpDetectorDNNOpenCV() [3/3]

vpDetectorDNNOpenCV::vpDetectorDNNOpenCV ( const std::string &  jsonPath,
void(*)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)  parsingMethod = postProcess_unimplemented 
)

Construct a new vpDetectorDNNOpenCV object from a JSON file and a potential parsing method.

Parameters
jsonPathThe JSON file permitting to initialize the detector.
parsingMethodIf the user chose to use a user-specified parsing method, the parsing method that must be used to parse the raw results of the DNN detection step.

Definition at line 176 of file vpDetectorDNNOpenCV.cpp.

References initFromJSON(), m_netConfig, setDetectionFilterSizeRatio(), and setParsingMethod().

◆ ~vpDetectorDNNOpenCV()

vpDetectorDNNOpenCV::~vpDetectorDNNOpenCV ( )
virtual

Destroy the vpDetectorDNNOpenCV object.

Definition at line 232 of file vpDetectorDNNOpenCV.cpp.

Member Function Documentation

◆ detect() [1/9]

bool vpDetectorDNNOpenCV::detect ( const cv::Mat &  I,
std::map< std::string, std::vector< DetectedFeatures2D >> &  output 
)
virtual

Object detection using OpenCV DNN module.

Parameters
I: Input image.
output: map where the name of the class is used as key and whose value is a vector of detected 2D features that belong to the class.
Returns
false if there is no detection.

Definition at line 388 of file vpDetectorDNNOpenCV.cpp.

References filterDetectionMultiClassInput(), m_applySizeFilterAfterNMS, m_blob, vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, m_dnnRes, m_img, m_indices, m_net, m_netConfig, m_outNames, and postProcess().

◆ detect() [2/9]

bool vpDetectorDNNOpenCV::detect ( const cv::Mat &  I,
std::vector< DetectedFeatures2D > &  output 
)
virtual

Object detection using OpenCV DNN module.

Parameters
I: Input image.
output: Vector of detections, whichever class they belong to.
Returns
false if there is no detection.

Definition at line 335 of file vpDetectorDNNOpenCV.cpp.

References filterDetectionMultiClassInput(), m_applySizeFilterAfterNMS, m_blob, vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, m_dnnRes, m_img, m_indices, m_net, m_netConfig, m_outNames, and postProcess().

◆ detect() [3/9]

bool vpDetectorDNNOpenCV::detect ( const cv::Mat &  I,
std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &  output 
)
virtual

Object detection using OpenCV DNN module.

Parameters
I: Input image.
output: vector of pairs <name_of_the_class, vector_of_detections>
Returns
false if there is no detection.

Definition at line 444 of file vpDetectorDNNOpenCV.cpp.

References detect().

◆ detect() [4/9]

bool vpDetectorDNNOpenCV::detect ( const vpImage< unsigned char > &  I,
std::map< std::string, std::vector< DetectedFeatures2D >> &  output 
)
virtual

Object detection using OpenCV DNN module.

Warning
Classical object detection network uses as input 3-channels. Grayscale image is converted to color image.
Parameters
I: Input image.
outputmap where the name of the class is used as key and whose value is a vector of detected 2D features that belong to the class.
Returns
false if there is no detection.

Definition at line 259 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_I_color.

◆ detect() [5/9]

bool vpDetectorDNNOpenCV::detect ( const vpImage< unsigned char > &  I,
std::vector< DetectedFeatures2D > &  output 
)
virtual

Object detection using OpenCV DNN module.

Warning
Classical object detection network uses as input 3-channels. Grayscale image is converted to color image.
Parameters
IInput image.
outputVector of detections, whichever class they belong to.
Returns
false if there is no detection, true otherwise.
Examples
tutorial-dnn-object-detection-live.cpp, and tutorial-megapose-live-single-object-tracking.cpp.

Definition at line 243 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), and m_I_color.

Referenced by detect().

◆ detect() [6/9]

bool vpDetectorDNNOpenCV::detect ( const vpImage< unsigned char > &  I,
std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &  output 
)
virtual

Object detection using OpenCV DNN module.

Warning
Classical object detection network uses as input 3-channels. Grayscale image is converted to color image.
Parameters
I: Input image.
outputvector of pairs <name_of_the_class, vector_of_detections>
Returns
false if there is no detection.

Definition at line 275 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_I_color.

◆ detect() [7/9]

bool vpDetectorDNNOpenCV::detect ( const vpImage< vpRGBa > &  I,
std::map< std::string, std::vector< DetectedFeatures2D >> &  output 
)
virtual

Object detection using OpenCV DNN module.

Warning
OpenCV DNN module uses cv::Mat as input. Grayscale image is converted to color image.
Parameters
I: Input image.
outputmap where the name of the class is used as key and whose value is a vector of detected 2D features that belong to the class.
Returns
false if there is no detection.

Definition at line 307 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_img.

◆ detect() [8/9]

bool vpDetectorDNNOpenCV::detect ( const vpImage< vpRGBa > &  I,
std::vector< DetectedFeatures2D > &  output 
)
virtual

Object detection using OpenCV DNN module.

Warning
OpenCV DNN module uses cv::Mat as input. vpImage<vpRGBa> is converted to cv::Mat.
Parameters
I: Input image.
outputVector of detections, whichever class they belong to.
Returns
false if there is no detection, true otherwise.

Definition at line 291 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_img.

◆ detect() [9/9]

bool vpDetectorDNNOpenCV::detect ( const vpImage< vpRGBa > &  I,
std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &  output 
)
virtual

Object detection using OpenCV DNN module.

Parameters
I: Input image.
outputvector of pairs <name_of_the_class, vector_of_detections>
Returns
false if there is no detection, true otherwise.

Definition at line 321 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_img.

◆ dnnResultsParsingTypeFromString()

vpDetectorDNNOpenCV::DNNResultsParsingType vpDetectorDNNOpenCV::dnnResultsParsingTypeFromString ( const std::string &  name)
static

Cast a name of class of parsing method into a vpDetectorDNNOpenCV::DNNResultsParsingType. The naming convention is the following:

Examples
tutorial-dnn-object-detection-live.cpp, and tutorial-megapose-live-single-object-tracking.cpp.

Definition at line 113 of file vpDetectorDNNOpenCV.cpp.

References COUNT, dnnResultsParsingTypeToString(), and vpIoTools::toLowerCase().

◆ dnnResultsParsingTypeToString()

std::string vpDetectorDNNOpenCV::dnnResultsParsingTypeToString ( const DNNResultsParsingType type)
static

Cast a vpDetectorDNNOpenCV::DNNResultsParsingType into its name, in std::string format. The naming convention is the following:

  • only lowercases
  • the underscores '_' are replaced by hyphens '-'.
Parameters
typethe type of parsing method to apply to interpret the DNN inference raw results.
Returns
std::string: the name of the type of parsing method to apply.

Definition at line 67 of file vpDetectorDNNOpenCV.cpp.

References COUNT, FASTER_RCNN, RESNET_10, SSD_MOBILENET, USER_SPECIFIED, YOLO_V3, YOLO_V4, YOLO_V5, YOLO_V7, and YOLO_V8.

Referenced by dnnResultsParsingTypeFromString(), getAvailableDnnResultsParsingTypes(), setParsingMethod(), and vpDetectorDNNOpenCV::NetConfig::toString().

◆ filterDetectionMultiClassInput() [1/2]

std::map< std::string, std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D > > vpDetectorDNNOpenCV::filterDetectionMultiClassInput ( const std::map< std::string, std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D >> &  detected_features,
const double  minRatioOfAreaOk 
)
protected

Return a new map <class, vector_corresponding_detections> where the area of each detection belonging to one class is in the range [average_area(class) x minRatioOfAreaOk ; average_area(class) / minRatioOfAreaOk ] .

Parameters
detected_featuresThe original detected features, that can contains several classes.
minRatioOfAreaOkThe minimum ratio of area a feature bounding box must reach to be kept in the resulting list of features. Value between 0 and 1.0.
Returns
std::map<std::string, std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D>> A new filtered map where each detection belonging to a class respect the area criterion.

Definition at line 660 of file vpDetectorDNNOpenCV.cpp.

References filterDetectionSingleClassInput().

◆ filterDetectionMultiClassInput() [2/2]

std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D > vpDetectorDNNOpenCV::filterDetectionMultiClassInput ( const std::vector< DetectedFeatures2D > &  detected_features,
const double  minRatioOfAreaOk 
)
protected

Return a new vector, ordered by vpDetectorDNNOpenCV::DetectedFeatures2D::m_cls , where the area of each detection belonging to one class is in the range [average_area(class) x minRatioOfAreaOk ; average_area(class) / minRatioOfAreaOk ] .

Parameters
detected_featuresThe original list of detected features, that can contains several classes.
minRatioOfAreaOkThe minimum ratio of area a feature bounding box must reach to be kept in the resulting list of features. Value between 0 and 1.0.
Returns
std::vector<vpDetectorDNNOpenCV::DetectedFeatures2> The filtered list of features, ordered by vpDetectorDNNOpenCV::DetectedFeatures2D::m_cls in ascending order, where only the features respecting the area criterion are kept.

Definition at line 568 of file vpDetectorDNNOpenCV.cpp.

References vpException::badValue, vpRect::getArea(), vpDetectorDNNOpenCV::DetectedFeatures2D::getBoundingBox(), and vpDetectorDNNOpenCV::DetectedFeatures2D::getClassId().

Referenced by detect().

◆ filterDetectionSingleClassInput()

std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D > vpDetectorDNNOpenCV::filterDetectionSingleClassInput ( const std::vector< DetectedFeatures2D > &  detected_features,
const double  minRatioOfAreaOk 
)
protected

Return a new vector of detected features whose area is greater or equal to the average area x minRatioOfAreaOk. This method assumes that detected_features contains detections of a single class.

Parameters
detected_featuresThe original list of detected features, belonging to the same class.
minRatioOfAreaOkThe minimum ratio of area a feature bounding box must reach to be kept in the resulting list of features. Value between 0 and 1.0.
Returns
std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D> The resulting list of features, that only contains the features whose area is in the range [average area x minRatioOfAreaOk ; average area / minRatioOfAreaOk ].

Definition at line 534 of file vpDetectorDNNOpenCV.cpp.

Referenced by filterDetectionMultiClassInput().

◆ getAvailableDnnResultsParsingTypes()

std::string vpDetectorDNNOpenCV::getAvailableDnnResultsParsingTypes ( )
static

Get the list of the parsing methods / types of DNNs supported by the vpDetectorDNNOpenCV class.

Returns
std::string The list of the supported parsing methods / types of DNNs.
Examples
tutorial-dnn-object-detection-live.cpp.

Definition at line 48 of file vpDetectorDNNOpenCV.cpp.

References COUNT, and dnnResultsParsingTypeToString().

◆ getNetConfig()

const NetConfig& vpDetectorDNNOpenCV::getNetConfig ( ) const
inline

◆ getOutputsNames()

std::vector< cv::String > vpDetectorDNNOpenCV::getOutputsNames ( )
protected

Get the names of the output layers of the DNN.

Returns
std::vector<cv::String>

Definition at line 460 of file vpDetectorDNNOpenCV.cpp.

References m_net.

Referenced by readNet().

◆ initFromJSON()

void vpDetectorDNNOpenCV::initFromJSON ( const std::string &  jsonPath)
Parameters
jsonPath
Examples
tutorial-dnn-object-detection-live.cpp.

Definition at line 190 of file vpDetectorDNNOpenCV.cpp.

References vpException::ioError, m_netConfig, and readNet().

Referenced by vpDetectorDNNOpenCV().

◆ parseClassNamesFile()

std::vector< std::string > vpDetectorDNNOpenCV::parseClassNamesFile ( const std::string &  filename)
static

Parse the designated file that contains the list of the classes the network can detect. The class names must either be indicated in an array of string in YAML format, or one name by row (without quotes) as described in NetConfig::parseClassNamesFile().

Parameters
filenameThe path towards the file.
Returns
std::vector<std::string> The list of class names.
See also
NetConfig::parseClassNamesFile().

Definition at line 138 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::NetConfig::parseClassNamesFile().

◆ postProcess()

void vpDetectorDNNOpenCV::postProcess ( DetectionCandidates proposals)
protected

Post-process the raw results of the DNN. Call the post-process method corresponding to the vpDetectorDNNOpenCV::DNNResultsParsingType that will extract the data stored as a matrix. Then, perform Non-Maximum Suppression to remove overlapping detections.

Parameters
[in,out]proposals: Input/output that will contains all the detection candidates.

Definition at line 482 of file vpDetectorDNNOpenCV.cpp.

References vpException::badValue, FASTER_RCNN, vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, m_dnnRes, m_indices, m_netConfig, m_parsingMethod, postProcess_FasterRCNN(), postProcess_ResNet_10(), postProcess_SSD_MobileNet(), postProcess_YoloV3_V4(), postProcess_YoloV5_V7(), postProcess_YoloV8(), RESNET_10, SSD_MOBILENET, USER_SPECIFIED, YOLO_V3, YOLO_V4, YOLO_V5, YOLO_V7, and YOLO_V8.

Referenced by detect().

◆ postProcess_FasterRCNN()

void vpDetectorDNNOpenCV::postProcess_FasterRCNN ( DetectionCandidates proposals,
std::vector< cv::Mat > &  dnnRes,
const NetConfig netConfig 
)
protected

Post-process the raw results of a Faster-RCNN-type DNN. Extract the data stored as a matrix. The network produces output blob with a shape 1x1xNx7 where N is a number of detections and an every detection is a vector of values [batchId, classId, confidence, left, top, right, bottom]

Parameters
proposals: input/output that will contains all the detection candidates.
dnnResraw results of the vpDetectorDNNOpenCV::detect step.
netConfigthe configuration of the network, to know for instance the DNN input size.

Definition at line 873 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ postProcess_ResNet_10()

void vpDetectorDNNOpenCV::postProcess_ResNet_10 ( DetectionCandidates proposals,
std::vector< cv::Mat > &  dnnRes,
const NetConfig netConfig 
)
protected

Post-process the raw results of a ResNet-10-type DNN. Extract the data stored as a matrix. The network produces output blob with a shape 1x1xNx7 where N is a number of detections and an every detection is a vector of values [batchId, classId, confidence, left, top, right, bottom]

Parameters
proposals: input/output that will contains all the detection candidates.
dnnResraw results of the vpDetectorDNNOpenCV::detect step.
netConfigthe configuration of the network, to know for instance the DNN input size.

Definition at line 976 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ postProcess_SSD_MobileNet()

void vpDetectorDNNOpenCV::postProcess_SSD_MobileNet ( DetectionCandidates proposals,
std::vector< cv::Mat > &  dnnRes,
const NetConfig netConfig 
)
protected

Post-process the raw results of a SSD-MobileNet-type DNN. Extract the data stored as a matrix. The network produces 2 outputs blob:

  • scores with dimensions 1xNxC
  • 'boxes' with dimensions 1xNx4 where N is a number of detections and C is the number of classes (with BACKGROUND as classId = 0).
Parameters
proposals: input/output that will contains all the detection candidates.
dnnResraw results of the vpDetectorDNNOpenCV::detect step.
netConfigthe configuration of the network, to know for instance the DNN input size.

Definition at line 915 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, m_img, and m_outNames.

Referenced by postProcess().

◆ postProcess_unimplemented()

void vpDetectorDNNOpenCV::postProcess_unimplemented ( DetectionCandidates proposals,
std::vector< cv::Mat > &  dnnRes,
const NetConfig netConfig 
)
staticprotected

Method throwing a vpException::functionNotImplementedError . It is called if the user set the vpDetectorDNNOpenCV::NetConfig::m_parsingMethodType to USER_DEFINED but didn't set the parsing method.

Parameters
proposals: input/output that will contains all the detection candidates.
dnnResraw results of the vpDetectorDNNOpenCV::detect step.
netConfigthe configuration of the network, to know for instance the DNN input size.

Definition at line 1009 of file vpDetectorDNNOpenCV.cpp.

References vpException::functionNotImplementedError.

◆ postProcess_YoloV3_V4()

void vpDetectorDNNOpenCV::postProcess_YoloV3_V4 ( DetectionCandidates proposals,
std::vector< cv::Mat > &  dnnRes,
const NetConfig netConfig 
)
protected

Post-process the raw results of a Yolov3-type or YoloV4-type DNN. Extract the data stored as a matrix. They are stored as follow: [batchsize][1:nb_proposals][1:5+nb_classes] Where a detection proposal consists of: [center_x; center_y; width; height; objectness; score_class_0; ...; score_last_class] where center_x € [0; 1] and center_y € [0; 1] which correspond to the ratio of the position of the center of the bbox with regard to the total width/height of the image.

Parameters
proposals: input/output that will contains all the detection candidates.
dnnResraw results of the vpDetectorDNNOpenCV::detect step.
netConfigthe configuration of the network, to know for instance the DNN input size.

cx,cy,w,h,box_score,class_score

cx

cy

w

h

Definition at line 682 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ postProcess_YoloV5_V7()

void vpDetectorDNNOpenCV::postProcess_YoloV5_V7 ( DetectionCandidates proposals,
std::vector< cv::Mat > &  dnnRes,
const NetConfig netConfig 
)
protected

Post-process the raw results of a YoloV5-type or a YoloV7-type DNN. Extract the data stored as a matrix. They are stored as follow: [batchsize][1:num_proposals][1:5+nb_classes] Where a detection proposal consists of: [center_x; center_y; width; height; objectness; score_class_0; ...; score_last_class]

Parameters
proposals: input/output that will contains all the detection candidates.
dnnResraw results of the vpDetectorDNNOpenCV::detect step.
netConfigthe configuration of the network, to know for instance the DNN input size.

cx,cy,w,h,box_score,class_score

cx

cy

w

h

Definition at line 745 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ postProcess_YoloV8()

void vpDetectorDNNOpenCV::postProcess_YoloV8 ( DetectionCandidates proposals,
std::vector< cv::Mat > &  dnnRes,
const NetConfig netConfig 
)
protected

Post-process the raw results of a YoloV8-type DNN. Extract the data stored as a matrix. They are stored as follow: [batchsize][1:5+nb_classes][1:8400] Where 8400 is the number of detection proposals and a detection proposal consists of: [center_x; center_y; width; height; score_class_0; ...; score_last_class]^T

Parameters
proposals: input/output that will contains all the detection candidates.
dnnResraw results of the vpDetectorDNNOpenCV::detect step.
netConfigthe configuration of the network, to know for instance the DNN input size.

cx,cy,w,h,box_score,class_score

cx

cy

w

h

Definition at line 810 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ readNet()

void vpDetectorDNNOpenCV::readNet ( const std::string &  model,
const std::string &  config = "",
const std::string &  framework = "" 
)

Read a network, see OpenCV readNet documentation for more information.

Parameters
modelPath to a binary file of model containing trained weights. The following file extensions are expected for models from different frameworks:
configPath to a text file of model containing network configuration. It could be a file with the following extensions:
frameworkOptional name of an origin framework of the model. Automatically detected if it is not set.

Definition at line 1036 of file vpDetectorDNNOpenCV.cpp.

References getOutputsNames(), m_net, m_netConfig, and m_outNames.

Referenced by initFromJSON(), setNetConfig(), and vpDetectorDNNOpenCV().

◆ saveConfigurationInJSON()

void vpDetectorDNNOpenCV::saveConfigurationInJSON ( const std::string &  jsonPath) const

Save the network configuration in a JSON file.

Parameters
jsonPathPath towards the output JSON file .
Examples
tutorial-dnn-object-detection-live.cpp.

Definition at line 220 of file vpDetectorDNNOpenCV.cpp.

◆ setConfidenceThreshold()

void vpDetectorDNNOpenCV::setConfidenceThreshold ( const float &  confThreshold)

Set confidence threshold to filter the detections.

Parameters
confThresholdConfidence threshold between [0, 1]

Definition at line 1070 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

◆ setDetectionFilterSizeRatio()

void vpDetectorDNNOpenCV::setDetectionFilterSizeRatio ( const double &  sizeRatio)

Set the size ratio used in the filterDetection method. If <= 0., filterDetection is not used. The detections of class c for which the bbox area does not belong to [mean_class_c(Area) * sizeRatio ; mean_class_c(Area) / sizeRatio ] are removed from the list of detections.

Parameters
sizeRatiothe size ratio used in the filterDetection method. If <= 0., filterDetection is not used.

Definition at line 1087 of file vpDetectorDNNOpenCV.cpp.

References m_applySizeFilterAfterNMS, and m_netConfig.

Referenced by setNetConfig(), and vpDetectorDNNOpenCV().

◆ setInputSize()

void vpDetectorDNNOpenCV::setInputSize ( const int &  width,
const int &  height 
)

Set dimension to resize the image to the input blob.

Parameters
widthIf <= 0, blob width is set to image width
heightIf <= 0, blob height is set to image height

Definition at line 1104 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

◆ setMean()

void vpDetectorDNNOpenCV::setMean ( const double &  meanR,
const double &  meanG,
const double &  meanB 
)

Set mean subtraction values.

Parameters
meanRMean value for R-channel
meanGMean value for G-channel
meanBMean value for R-channel

Definition at line 1117 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

◆ setNetConfig()

void vpDetectorDNNOpenCV::setNetConfig ( const NetConfig config)

Configure the DNN (thresholds, input size, ...). If the DNN weights file is known, initialize the m_net by reading the weights.

Parameters
configthe desired configuration of the network
Examples
tutorial-dnn-object-detection-live.cpp.

Definition at line 1055 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig, readNet(), setDetectionFilterSizeRatio(), and setParsingMethod().

◆ setNMSThreshold()

void vpDetectorDNNOpenCV::setNMSThreshold ( const float &  nmsThreshold)

Set Non-Maximum Suppression threshold, used to filter multiple detections at approximatively the same location.

Parameters
nmsThresholdNon-Maximum Suppression threshold between [0, 1]

Definition at line 1078 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

◆ setParsingMethod()

void vpDetectorDNNOpenCV::setParsingMethod ( const DNNResultsParsingType typeParsingMethod,
void(*)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)  parsingMethod = postProcess_unimplemented 
)

Set the type of parsing method that must be used to interpret the raw results of the DNN detection.

Parameters
typeParsingMethodthe type of parsing method that must be used to interpret the raw results of the DNN detection.
parsingMethodif typeParsingMethod is equal to vpDetectorDNNOpenCV::USER_DEFINED , a function permitting to interpret the cv::Mat resulting from the DNN inference.

Definition at line 1160 of file vpDetectorDNNOpenCV.cpp.

References dnnResultsParsingTypeToString(), m_netConfig, m_parsingMethod, SSD_MOBILENET, YOLO_V7, and YOLO_V8.

Referenced by setNetConfig(), and vpDetectorDNNOpenCV().

◆ setPreferableBackend()

void vpDetectorDNNOpenCV::setPreferableBackend ( const int &  backendId)

Set preferable backend for inference computation. See OpenCV setPreferableBackend documentation for more information.

Parameters
backendIdBackend identifier

Definition at line 1125 of file vpDetectorDNNOpenCV.cpp.

References m_net.

◆ setPreferableTarget()

void vpDetectorDNNOpenCV::setPreferableTarget ( const int &  targetId)

Set preferable target for inference computation. See OpenCV setPreferableTarget documentation for more information.

Parameters
targetIdTarget identifier

Definition at line 1133 of file vpDetectorDNNOpenCV.cpp.

References m_net.

◆ setScaleFactor()

void vpDetectorDNNOpenCV::setScaleFactor ( const double &  scaleFactor)

Set scale factor to normalize the range of pixel values.

Definition at line 1138 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig, YOLO_V7, and YOLO_V8.

◆ setSwapRB()

void vpDetectorDNNOpenCV::setSwapRB ( const bool &  swapRB)

If true, swap R and B channel for mean subtraction. For instance when the network has been trained on RGB image format (OpenCV uses BGR convention).

Definition at line 1151 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

Friends And Related Function Documentation

◆ from_json

void from_json ( const json &  j,
vpDetectorDNNOpenCV network 
)
friend

Read the network configuration from JSON. All values are optional and if an argument is not present, the default value defined in the constructor is kept.

Parameters
jThe JSON object, resulting from the parsing of a JSON file.
networkThe network, that will be initialized from the JSON data.

Definition at line 514 of file vpDetectorDNNOpenCV.h.

◆ operator<<

std::ostream& operator<< ( std::ostream &  os,
const vpDetectorDNNOpenCV network 
)
friend

Definition at line 533 of file vpDetectorDNNOpenCV.h.

◆ to_json

void to_json ( json &  j,
const vpDetectorDNNOpenCV network 
)
friend

Parse the network configuration into JSON format.

Parameters
jThe JSON parser.
networkThe network we want to parse the configuration.

Definition at line 525 of file vpDetectorDNNOpenCV.h.

Member Data Documentation

◆ m_applySizeFilterAfterNMS

bool vpDetectorDNNOpenCV::m_applySizeFilterAfterNMS
protected

If true, filter the detections removing the ones for which the bbox does not respect area(bbox) € [mean_class(area) * ratio; mean_class(area) / ratio].

Definition at line 571 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), and setDetectionFilterSizeRatio().

◆ m_blob

cv::Mat vpDetectorDNNOpenCV::m_blob
protected

Buffer for the blob in input net.

Definition at line 573 of file vpDetectorDNNOpenCV.h.

Referenced by detect().

◆ m_dnnRes

std::vector<cv::Mat> vpDetectorDNNOpenCV::m_dnnRes
protected

Contains all output blobs for each layer specified in m_outNames.

Definition at line 587 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), and postProcess().

◆ m_I_color

vpImage<vpRGBa> vpDetectorDNNOpenCV::m_I_color
protected

Buffer for gray to RGBa image conversion.

Definition at line 575 of file vpDetectorDNNOpenCV.h.

Referenced by detect().

◆ m_img

cv::Mat vpDetectorDNNOpenCV::m_img
protected

◆ m_indices

std::vector<int> vpDetectorDNNOpenCV::m_indices
protected

Indices for NMS.

Definition at line 579 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), and postProcess().

◆ m_net

cv::dnn::Net vpDetectorDNNOpenCV::m_net
protected

DNN network.

Definition at line 581 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), getOutputsNames(), readNet(), setPreferableBackend(), and setPreferableTarget().

◆ m_netConfig

◆ m_outNames

std::vector<cv::String> vpDetectorDNNOpenCV::m_outNames
protected

Names of layers with unconnected outputs.

Definition at line 585 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), postProcess_SSD_MobileNet(), and readNet().

◆ m_parsingMethod

void(* vpDetectorDNNOpenCV::m_parsingMethod) (DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)
protected

Pointer towards the parsing method, used if m_parsingMethodType is equal to m_parsingMethodType::USER_SPECIFIED.

Definition at line 589 of file vpDetectorDNNOpenCV.h.

Referenced by postProcess(), and setParsingMethod().