#include <visp3/detection/vpDetectorDNNOpenCV.h>

Classes
class	DetectedFeatures2D

struct	DetectionCandidates

struct	NetConfig

Public Types
enum	DNNResultsParsingType { USER_SPECIFIED = 0 , FASTER_RCNN = 1 , SSD_MOBILENET = 2 , RESNET_10 = 3 , YOLO_V3 = 4 , YOLO_V4 = 5 , YOLO_V5 = 6 , YOLO_V7 = 7 , YOLO_V8 = 8 , COUNT = 9 }

typedef enum vpDetectorDNNOpenCV::DNNResultsParsingType	DNNResultsParsingType

typedef struct vpDetectorDNNOpenCV::DetectionCandidates	DetectionCandidates

typedef class vpDetectorDNNOpenCV::DetectedFeatures2D	DetectedFeatures2D

typedef class vpDetectorDNNOpenCV::NetConfig	NetConfig

Public Member Functions
	vpDetectorDNNOpenCV ()

	vpDetectorDNNOpenCV (const NetConfig &config, const DNNResultsParsingType &typeParsingMethod, void(*parsingMethod)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)=postProcess_unimplemented)

	vpDetectorDNNOpenCV (const std::string &jsonPath, void(*parsingMethod)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)=postProcess_unimplemented)

void	initFromJSON (const std::string &jsonPath)

void	saveConfigurationInJSON (const std::string &jsonPath) const

virtual	~vpDetectorDNNOpenCV ()

virtual bool	detect (const vpImage< unsigned char > &I, std::vector< DetectedFeatures2D > &output)

virtual bool	detect (const vpImage< unsigned char > &I, std::map< std::string, std::vector< DetectedFeatures2D >> &output)

virtual bool	detect (const vpImage< unsigned char > &I, std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &output)

virtual bool	detect (const vpImage< vpRGBa > &I, std::vector< DetectedFeatures2D > &output)

virtual bool	detect (const vpImage< vpRGBa > &I, std::map< std::string, std::vector< DetectedFeatures2D >> &output)

virtual bool	detect (const vpImage< vpRGBa > &I, std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &output)

virtual bool	detect (const cv::Mat &I, std::vector< DetectedFeatures2D > &output)

virtual bool	detect (const cv::Mat &I, std::map< std::string, std::vector< DetectedFeatures2D >> &output)

virtual bool	detect (const cv::Mat &I, std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &output)

void	readNet (const std::string &model, const std::string &config="", const std::string &framework="")

void	setNetConfig (const NetConfig &config)

void	setConfidenceThreshold (const float &confThreshold)

void	setNMSThreshold (const float &nmsThreshold)

void	setDetectionFilterSizeRatio (const double &sizeRatio)

void	setInputSize (const int &width, const int &height)

void	setMean (const double &meanR, const double &meanG, const double &meanB)

void	setPreferableBackend (const int &backendId)

void	setPreferableTarget (const int &targetId)

void	setScaleFactor (const double &scaleFactor)

void	setSwapRB (const bool &swapRB)

void	setParsingMethod (const DNNResultsParsingType &typeParsingMethod, void(*parsingMethod)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)=postProcess_unimplemented)

const NetConfig &	getNetConfig () const

Static Public Member Functions
static std::string	getAvailableDnnResultsParsingTypes ()

static std::string	dnnResultsParsingTypeToString (const DNNResultsParsingType &type)

static DNNResultsParsingType	dnnResultsParsingTypeFromString (const std::string &name)

static std::vector< std::string >	parseClassNamesFile (const std::string &filename)

Protected Member Functions
std::vector< cv::String >	getOutputsNames ()

std::vector< DetectedFeatures2D >	filterDetectionSingleClassInput (const std::vector< DetectedFeatures2D > &detected_features, const double minRatioOfAreaOk)

std::vector< DetectedFeatures2D >	filterDetectionMultiClassInput (const std::vector< DetectedFeatures2D > &detected_features, const double minRatioOfAreaOk)

std::map< std::string, std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D > >	filterDetectionMultiClassInput (const std::map< std::string, std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D >> &detected_features, const double minRatioOfAreaOk)

void	postProcess (DetectionCandidates &proposals)

void	postProcess_YoloV3_V4 (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)

void	postProcess_YoloV5_V7 (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)

void	postProcess_YoloV8 (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)

void	postProcess_FasterRCNN (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)

void	postProcess_SSD_MobileNet (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)

void	postProcess_ResNet_10 (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)

Static Protected Member Functions
static void	postProcess_unimplemented (DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)

Protected Attributes
bool	m_applySizeFilterAfterNMS

cv::Mat	m_blob

vpImage< vpRGBa >	m_I_color

cv::Mat	m_img

std::vector< int >	m_indices

cv::dnn::Net	m_net

NetConfig	m_netConfig

std::vector< cv::String >	m_outNames

std::vector< cv::Mat >	m_dnnRes

void(*	m_parsingMethod )(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)

Friends
void	from_json (const nlohmann::json &j, vpDetectorDNNOpenCV &network)

void	to_json (nlohmann::json &j, const vpDetectorDNNOpenCV &network)

std::ostream &	operator<< (std::ostream &os, const vpDetectorDNNOpenCV &network)

Detailed Description

This class is a wrapper over the OpenCV DNN module and specialized to handle object detection task.

This class supports the following networks dedicated to object detection:

Faster-RCNN, see usage to detect objects belonging to the COCO dataset using Faster-RCNN network
SSD MobileNet, see usage to detect objects belonging to the COCO dataset using MobileNet SSD network
ResNet 10, see usage for Face detection
Yolo v3, see usage to detect objects belonging to the COCO dataset using Yolo v3 network
Yolo v4, see usage to detect objects belonging to the COCO dataset using Yolo v4 network
Yolo v5, see usage to detect objects belonging to the COCO dataset using Yolo v5 network
Yolo v7, see usage to detect objects belonging to the COCO dataset using Yolo v7 network
Yolo v8, see usage to detect objects belonging to the COCO dataset using Yolo v8 network

This class can be initialized from a JSON file if ViSP has been compiled with NLOHMANN JSON (see JSON for modern C++ to see how to do it). Examples of such JSON files can be found in the tutorial folder.

See also: Tutorial: Deep learning object detection

Examples: tutorial-dnn-object-detection-live.cpp, and tutorial-megapose-live-single-object-tracking.cpp.

Definition at line 83 of file vpDetectorDNNOpenCV.h.

Member Typedef Documentation

◆ DetectedFeatures2D

typedef class vpDetectorDNNOpenCV::DetectedFeatures2D vpDetectorDNNOpenCV::DetectedFeatures2D

◆ DetectionCandidates

typedef struct vpDetectorDNNOpenCV::DetectionCandidates vpDetectorDNNOpenCV::DetectionCandidates

◆ DNNResultsParsingType

typedef enum vpDetectorDNNOpenCV::DNNResultsParsingType vpDetectorDNNOpenCV::DNNResultsParsingType

◆ NetConfig

typedef class vpDetectorDNNOpenCV::NetConfig vpDetectorDNNOpenCV::NetConfig

Member Enumeration Documentation

◆ DNNResultsParsingType

enum vpDetectorDNNOpenCV::DNNResultsParsingType

Enumeration listing the types of DNN for which the vpDetectorDNNOpenCV furnishes the methods permitting to parse the raw detection data into vpDetectorDNNOpenCV::DetectedFeatures2D .

Enumerator
USER_SPECIFIED	The user will give a pointer towards the parsing method to use to parse the raw data resulting from the detection step.
FASTER_RCNN	The vpDetectorDNNOpenCV object will use the parsing method corresponding to a Faster-RCNN DNN. See vpDetectorDNNOpenCV::postProcess_FasterRCNN for more information.
SSD_MOBILENET	The vpDetectorDNNOpenCV object will use the parsing method corresponding to a SSD MobileNet DNN. See vpDetectorDNNOpenCV::postProcess_SSD_MobileNet for more information.
RESNET_10	The vpDetectorDNNOpenCV object will use the parsing method corresponding to a ResNet 10 DNN. See vpDetectorDNNOpenCV::postProcess_ResNet_10 for more information.
YOLO_V3	The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV3 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV3_V4 for more information.
YOLO_V4	The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV4 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV3_V4 for more information.
YOLO_V5	The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV5 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV5_V7 for more information.
YOLO_V7	The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV7 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV5_V7 for more information.
YOLO_V8	The vpDetectorDNNOpenCV object will use the parsing method corresponding to a YoloV8 DNN. See vpDetectorDNNOpenCV::postProcess_YoloV8 for more information.
COUNT	The number of parsing method that come along with the vpDetectorDNNOpenCV class.

Definition at line 91 of file vpDetectorDNNOpenCV.h.

Constructor & Destructor Documentation

◆ vpDetectorDNNOpenCV() [1/3]

vpDetectorDNNOpenCV::vpDetectorDNNOpenCV ( )

Definition at line 148 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig, and setDetectionFilterSizeRatio().

◆ vpDetectorDNNOpenCV() [2/3]

vpDetectorDNNOpenCV::vpDetectorDNNOpenCV	(	const NetConfig &	config,
		const DNNResultsParsingType &	typeParsingMethod,
		void(*)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)	parsingMethod = `postProcess_unimplemented`
	)

Construct a new vpDetectorDNNOpenCV object and, if the model file has been given to the config object, read the net by calling vpDetectorDNNOpenCV::readNet.

Parameters

config	The network configuration.
typeParsingMethod	The type of parsing method that must be used to parse the raw results of the DNN detection step.
parsingMethod	If typeParsingMethod is set to vpDetectorDNNOpenCV::DNNResultsParsingType::USER_SPECIFIED, the parsing method that must be used to parse the raw results of the DNN detection step.

Definition at line 163 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig, readNet(), setDetectionFilterSizeRatio(), and setParsingMethod().

◆ vpDetectorDNNOpenCV() [3/3]

vpDetectorDNNOpenCV::vpDetectorDNNOpenCV	(	const std::string &	jsonPath,
		void(*)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)	parsingMethod = `postProcess_unimplemented`
	)

Construct a new vpDetectorDNNOpenCV object from a JSON file and a potential parsing method.

Parameters

jsonPath	The JSON file permitting to initialize the detector.
parsingMethod	If the user chose to use a user-specified parsing method, the parsing method that must be used to parse the raw results of the DNN detection step.

Definition at line 184 of file vpDetectorDNNOpenCV.cpp.

References initFromJSON(), m_netConfig, setDetectionFilterSizeRatio(), and setParsingMethod().

◆ ~vpDetectorDNNOpenCV()

vpDetectorDNNOpenCV::~vpDetectorDNNOpenCV ( )

virtual

Destroy the vpDetectorDNNOpenCV object.

Definition at line 240 of file vpDetectorDNNOpenCV.cpp.

Member Function Documentation

◆ detect() [1/9]

bool vpDetectorDNNOpenCV::detect	(	const cv::Mat &	I,
		std::map< std::string, std::vector< DetectedFeatures2D >> &	output
	)

virtual

Object detection using OpenCV DNN module.

Parameters

I	: Input image.
output	: map where the name of the class is used as key and whose value is a vector of detected 2D features that belong to the class.

Returns: false if there is no detection.

Definition at line 396 of file vpDetectorDNNOpenCV.cpp.

References filterDetectionMultiClassInput(), m_applySizeFilterAfterNMS, m_blob, vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, m_dnnRes, m_img, m_indices, m_net, m_netConfig, m_outNames, and postProcess().

◆ detect() [2/9]

bool vpDetectorDNNOpenCV::detect	(	const cv::Mat &	I,
		std::vector< DetectedFeatures2D > &	output
	)

virtual

Object detection using OpenCV DNN module.

Parameters

I	: Input image.
output	: Vector of detections, whichever class they belong to.

Returns: false if there is no detection.

Definition at line 343 of file vpDetectorDNNOpenCV.cpp.

References filterDetectionMultiClassInput(), m_applySizeFilterAfterNMS, m_blob, vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, m_dnnRes, m_img, m_indices, m_net, m_netConfig, m_outNames, and postProcess().

◆ detect() [3/9]

bool vpDetectorDNNOpenCV::detect	(	const cv::Mat &	I,
		std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &	output
	)

virtual

Object detection using OpenCV DNN module.

Parameters

I	: Input image.
output	: vector of pairs <name_of_the_class, vector_of_detections>

Returns: false if there is no detection.

Definition at line 452 of file vpDetectorDNNOpenCV.cpp.

References detect().

◆ detect() [4/9]

bool vpDetectorDNNOpenCV::detect	(	const vpImage< unsigned char > &	I,
		std::map< std::string, std::vector< DetectedFeatures2D >> &	output
	)

virtual

Object detection using OpenCV DNN module.

Warning: Classical object detection network uses as input 3-channels. Grayscale image is converted to color image.

Parameters

I	: Input image.
output	map where the name of the class is used as key and whose value is a vector of detected 2D features that belong to the class.

Returns: false if there is no detection.

Definition at line 267 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_I_color.

◆ detect() [5/9]

bool vpDetectorDNNOpenCV::detect	(	const vpImage< unsigned char > &	I,
		std::vector< DetectedFeatures2D > &	output
	)

virtual

Object detection using OpenCV DNN module.

Warning: Classical object detection network uses as input 3-channels. Grayscale image is converted to color image.

Parameters

I	Input image.
output	Vector of detections, whichever class they belong to.

Returns: false if there is no detection, true otherwise.

Examples: tutorial-dnn-object-detection-live.cpp, and tutorial-megapose-live-single-object-tracking.cpp.

Definition at line 251 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), and m_I_color.

Referenced by detect().

◆ detect() [6/9]

bool vpDetectorDNNOpenCV::detect	(	const vpImage< unsigned char > &	I,
		std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &	output
	)

virtual

Object detection using OpenCV DNN module.

Warning: Classical object detection network uses as input 3-channels. Grayscale image is converted to color image.

Parameters

I	: Input image.
output	vector of pairs <name_of_the_class, vector_of_detections>

Returns: false if there is no detection.

Definition at line 283 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_I_color.

◆ detect() [7/9]

bool vpDetectorDNNOpenCV::detect	(	const vpImage< vpRGBa > &	I,
		std::map< std::string, std::vector< DetectedFeatures2D >> &	output
	)

virtual

Object detection using OpenCV DNN module.

Warning: OpenCV DNN module uses cv::Mat as input. Grayscale image is converted to color image.

Parameters

I	: Input image.
output	map where the name of the class is used as key and whose value is a vector of detected 2D features that belong to the class.

Returns: false if there is no detection.

Definition at line 315 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_img.

◆ detect() [8/9]

bool vpDetectorDNNOpenCV::detect	(	const vpImage< vpRGBa > &	I,
		std::vector< DetectedFeatures2D > &	output
	)

virtual

Object detection using OpenCV DNN module.

Warning: OpenCV DNN module uses cv::Mat as input. vpImage<vpRGBa> is converted to cv::Mat.

Parameters

I	: Input image.
output	Vector of detections, whichever class they belong to.

Returns: false if there is no detection, true otherwise.

Definition at line 299 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_img.

◆ detect() [9/9]

bool vpDetectorDNNOpenCV::detect	(	const vpImage< vpRGBa > &	I,
		std::vector< std::pair< std::string, std::vector< DetectedFeatures2D >>> &	output
	)

virtual

Object detection using OpenCV DNN module.

Parameters

I	: Input image.
output	vector of pairs <name_of_the_class, vector_of_detections>

Returns: false if there is no detection, true otherwise.

Definition at line 329 of file vpDetectorDNNOpenCV.cpp.

References vpImageConvert::convert(), detect(), and m_img.

◆ dnnResultsParsingTypeFromString()

vpDetectorDNNOpenCV::DNNResultsParsingType vpDetectorDNNOpenCV::dnnResultsParsingTypeFromString ( const std::string & name )

static

Cast a name of class of parsing method into a vpDetectorDNNOpenCV::DNNResultsParsingType. The naming convention is the following:

only lowercases
the underscores '_' are replaced by hyphens '-'.
Parameters

name the name of the type of parsing method to apply.

Returns
vpDetectorDNNOpenCV::DNNResultsParsingType: the corresponding vpDetectorDNNOpenCV::DNNResultsParsingType.

Examples: tutorial-dnn-object-detection-live.cpp, and tutorial-megapose-live-single-object-tracking.cpp.

Definition at line 118 of file vpDetectorDNNOpenCV.cpp.

References COUNT, dnnResultsParsingTypeToString(), and vpIoTools::toLowerCase().

◆ dnnResultsParsingTypeToString()

std::string vpDetectorDNNOpenCV::dnnResultsParsingTypeToString ( const DNNResultsParsingType & type )

static

Cast a vpDetectorDNNOpenCV::DNNResultsParsingType into its name, in std::string format. The naming convention is the following:

only lowercases
the underscores '_' are replaced by hyphens '-'.

Parameters

type	the type of parsing method to apply to interpret the DNN inference raw results.

Returns: std::string: the name of the type of parsing method to apply.

Definition at line 72 of file vpDetectorDNNOpenCV.cpp.

References COUNT, FASTER_RCNN, RESNET_10, SSD_MOBILENET, USER_SPECIFIED, YOLO_V3, YOLO_V4, YOLO_V5, YOLO_V7, and YOLO_V8.

Referenced by dnnResultsParsingTypeFromString(), getAvailableDnnResultsParsingTypes(), setParsingMethod(), and vpDetectorDNNOpenCV::NetConfig::toString().

◆ filterDetectionMultiClassInput() [1/2]

std::map< std::string, std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D > > vpDetectorDNNOpenCV::filterDetectionMultiClassInput	(	const std::map< std::string, std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D >> &	detected_features,
		const double	minRatioOfAreaOk
	)

protected

Return a new map <class, vector_corresponding_detections> where the area of each detection belonging to one class is in the range [average_area(class) x minRatioOfAreaOk ; average_area(class) / minRatioOfAreaOk ] .

Parameters

detected_features	The original detected features, that can contains several classes.
minRatioOfAreaOk	The minimum ratio of area a feature bounding box must reach to be kept in the resulting list of features. Value between 0 and 1.0.

Returns: std::map<std::string, std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D>> A new filtered map where each detection belonging to a class respect the area criterion.

Definition at line 668 of file vpDetectorDNNOpenCV.cpp.

References filterDetectionSingleClassInput().

◆ filterDetectionMultiClassInput() [2/2]

std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D > vpDetectorDNNOpenCV::filterDetectionMultiClassInput	(	const std::vector< DetectedFeatures2D > &	detected_features,
		const double	minRatioOfAreaOk
	)

protected

Return a new vector, ordered by vpDetectorDNNOpenCV::DetectedFeatures2D::m_cls , where the area of each detection belonging to one class is in the range [average_area(class) x minRatioOfAreaOk ; average_area(class) / minRatioOfAreaOk ] .

Parameters

detected_features	The original list of detected features, that can contains several classes.
minRatioOfAreaOk	The minimum ratio of area a feature bounding box must reach to be kept in the resulting list of features. Value between 0 and 1.0.

Returns: std::vector<vpDetectorDNNOpenCV::DetectedFeatures2> The filtered list of features, ordered by vpDetectorDNNOpenCV::DetectedFeatures2D::m_cls in ascending order, where only the features respecting the area criterion are kept.

Definition at line 576 of file vpDetectorDNNOpenCV.cpp.

References vpException::badValue, vpRect::getArea(), vpDetectorDNNOpenCV::DetectedFeatures2D::getBoundingBox(), and vpDetectorDNNOpenCV::DetectedFeatures2D::getClassId().

Referenced by detect().

◆ filterDetectionSingleClassInput()

std::vector< vpDetectorDNNOpenCV::DetectedFeatures2D > vpDetectorDNNOpenCV::filterDetectionSingleClassInput	(	const std::vector< DetectedFeatures2D > &	detected_features,
		const double	minRatioOfAreaOk
	)

protected

Return a new vector of detected features whose area is greater or equal to the average area x minRatioOfAreaOk. This method assumes that detected_features contains detections of a single class.

Parameters

detected_features	The original list of detected features, belonging to the same class.
minRatioOfAreaOk	The minimum ratio of area a feature bounding box must reach to be kept in the resulting list of features. Value between 0 and 1.0.

Returns: std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D> The resulting list of features, that only contains the features whose area is in the range [average area x minRatioOfAreaOk ; average area / minRatioOfAreaOk ].

Definition at line 542 of file vpDetectorDNNOpenCV.cpp.

Referenced by filterDetectionMultiClassInput().

◆ getAvailableDnnResultsParsingTypes()

BEGIN_VISP_NAMESPACE std::string vpDetectorDNNOpenCV::getAvailableDnnResultsParsingTypes ( )

static

Get the list of the parsing methods / types of DNNs supported by the vpDetectorDNNOpenCV class.

Returns: std::string The list of the supported parsing methods / types of DNNs.

Examples: tutorial-dnn-object-detection-live.cpp.

Definition at line 53 of file vpDetectorDNNOpenCV.cpp.

References COUNT, and dnnResultsParsingTypeToString().

◆ getNetConfig()

const NetConfig& vpDetectorDNNOpenCV::getNetConfig ( ) const

inline

Examples: tutorial-dnn-object-detection-live.cpp.

Definition at line 506 of file vpDetectorDNNOpenCV.h.

◆ getOutputsNames()

std::vector< cv::String > vpDetectorDNNOpenCV::getOutputsNames ( )

protected

Get the names of the output layers of the DNN.

Returns: std::vector<cv::String>

Definition at line 468 of file vpDetectorDNNOpenCV.cpp.

References m_net.

Referenced by readNet().

◆ initFromJSON()

void vpDetectorDNNOpenCV::initFromJSON ( const std::string & jsonPath )

Parameters

jsonPath

Examples: tutorial-dnn-object-detection-live.cpp.

Definition at line 198 of file vpDetectorDNNOpenCV.cpp.

References vpException::ioError, m_netConfig, and readNet().

Referenced by vpDetectorDNNOpenCV().

◆ parseClassNamesFile()

std::vector< std::string > vpDetectorDNNOpenCV::parseClassNamesFile ( const std::string & filename )

static

Parse the designated file that contains the list of the classes the network can detect. The class names must either be indicated in an array of string in YAML format, or one name by row (without quotes) as described in NetConfig::parseClassNamesFile().

Parameters

filename The path towards the file.

Returns: std::vector<std::string> The list of class names.

See also: NetConfig::parseClassNamesFile().

Definition at line 143 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::NetConfig::parseClassNamesFile().

◆ postProcess()

void vpDetectorDNNOpenCV::postProcess ( DetectionCandidates & proposals )

protected

Post-process the raw results of the DNN. Call the post-process method corresponding to the vpDetectorDNNOpenCV::DNNResultsParsingType that will extract the data stored as a matrix. Then, perform Non-Maximum Suppression to remove overlapping detections.

Parameters

[in,out] proposals : Input/output that will contains all the detection candidates.

Definition at line 490 of file vpDetectorDNNOpenCV.cpp.

References vpException::badValue, FASTER_RCNN, vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, m_dnnRes, m_indices, m_netConfig, m_parsingMethod, postProcess_FasterRCNN(), postProcess_ResNet_10(), postProcess_SSD_MobileNet(), postProcess_YoloV3_V4(), postProcess_YoloV5_V7(), postProcess_YoloV8(), RESNET_10, SSD_MOBILENET, USER_SPECIFIED, YOLO_V3, YOLO_V4, YOLO_V5, YOLO_V7, and YOLO_V8.

Referenced by detect().

◆ postProcess_FasterRCNN()

void vpDetectorDNNOpenCV::postProcess_FasterRCNN	(	DetectionCandidates &	proposals,
		std::vector< cv::Mat > &	dnnRes,
		const NetConfig &	netConfig
	)

protected

Post-process the raw results of a Faster-RCNN-type DNN. Extract the data stored as a matrix. The network produces output blob with a shape 1x1xNx7 where N is a number of detections and an every detection is a vector of values [batchId, classId, confidence, left, top, right, bottom]

Parameters

proposals	: input/output that will contains all the detection candidates.
dnnRes	raw results of the vpDetectorDNNOpenCV::detect step.
netConfig	the configuration of the network, to know for instance the DNN input size.

Definition at line 881 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ postProcess_ResNet_10()

void vpDetectorDNNOpenCV::postProcess_ResNet_10	(	DetectionCandidates &	proposals,
		std::vector< cv::Mat > &	dnnRes,
		const NetConfig &	netConfig
	)

protected

Post-process the raw results of a ResNet-10-type DNN. Extract the data stored as a matrix. The network produces output blob with a shape 1x1xNx7 where N is a number of detections and an every detection is a vector of values [batchId, classId, confidence, left, top, right, bottom]

Parameters

proposals	: input/output that will contains all the detection candidates.
dnnRes	raw results of the vpDetectorDNNOpenCV::detect step.
netConfig	the configuration of the network, to know for instance the DNN input size.

Definition at line 984 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ postProcess_SSD_MobileNet()

void vpDetectorDNNOpenCV::postProcess_SSD_MobileNet	(	DetectionCandidates &	proposals,
		std::vector< cv::Mat > &	dnnRes,
		const NetConfig &	netConfig
	)

protected

Post-process the raw results of a SSD-MobileNet-type DNN. Extract the data stored as a matrix. The network produces 2 outputs blob:

scores with dimensions 1xNxC
'boxes' with dimensions 1xNx4 where N is a number of detections and C is the number of classes (with BACKGROUND as classId = 0).

Parameters

proposals	: input/output that will contains all the detection candidates.
dnnRes	raw results of the vpDetectorDNNOpenCV::detect step.
netConfig	the configuration of the network, to know for instance the DNN input size.

Definition at line 923 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, m_img, and m_outNames.

Referenced by postProcess().

◆ postProcess_unimplemented()

void vpDetectorDNNOpenCV::postProcess_unimplemented	(	DetectionCandidates &	proposals,
		std::vector< cv::Mat > &	dnnRes,
		const NetConfig &	netConfig
	)

staticprotected

Method throwing a vpException::functionNotImplementedError . It is called if the user set the vpDetectorDNNOpenCV::NetConfig::m_parsingMethodType to USER_DEFINED but didn't set the parsing method.

Parameters

proposals	: input/output that will contains all the detection candidates.
dnnRes	raw results of the vpDetectorDNNOpenCV::detect step.
netConfig	the configuration of the network, to know for instance the DNN input size.

Definition at line 1017 of file vpDetectorDNNOpenCV.cpp.

References vpException::functionNotImplementedError.

◆ postProcess_YoloV3_V4()

void vpDetectorDNNOpenCV::postProcess_YoloV3_V4	(	DetectionCandidates &	proposals,
		std::vector< cv::Mat > &	dnnRes,
		const NetConfig &	netConfig
	)

protected

Post-process the raw results of a Yolov3-type or YoloV4-type DNN. Extract the data stored as a matrix. They are stored as follow: [batchsize][1:nb_proposals][1:5+nb_classes] Where a detection proposal consists of: [center_x; center_y; width; height; objectness; score_class_0; ...; score_last_class] where center_x € [0; 1] and center_y € [0; 1] which correspond to the ratio of the position of the center of the bbox with regard to the total width/height of the image.

Parameters

proposals	: input/output that will contains all the detection candidates.
dnnRes	raw results of the vpDetectorDNNOpenCV::detect step.
netConfig	the configuration of the network, to know for instance the DNN input size.

cx,cy,w,h,box_score,class_score

cx

cy

w

h

Definition at line 690 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ postProcess_YoloV5_V7()

void vpDetectorDNNOpenCV::postProcess_YoloV5_V7	(	DetectionCandidates &	proposals,
		std::vector< cv::Mat > &	dnnRes,
		const NetConfig &	netConfig
	)

protected

Post-process the raw results of a YoloV5-type or a YoloV7-type DNN. Extract the data stored as a matrix. They are stored as follow: [batchsize][1:num_proposals][1:5+nb_classes] Where a detection proposal consists of: [center_x; center_y; width; height; objectness; score_class_0; ...; score_last_class]

Parameters

proposals	: input/output that will contains all the detection candidates.
dnnRes	raw results of the vpDetectorDNNOpenCV::detect step.
netConfig	the configuration of the network, to know for instance the DNN input size.

cx,cy,w,h,box_score,class_score

cx

cy

w

h

Definition at line 753 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ postProcess_YoloV8()

void vpDetectorDNNOpenCV::postProcess_YoloV8	(	DetectionCandidates &	proposals,
		std::vector< cv::Mat > &	dnnRes,
		const NetConfig &	netConfig
	)

protected

Post-process the raw results of a YoloV8-type DNN. Extract the data stored as a matrix. They are stored as follow: [batchsize][1:5+nb_classes][1:8400] Where 8400 is the number of detection proposals and a detection proposal consists of: [center_x; center_y; width; height; score_class_0; ...; score_last_class]^T

Parameters

proposals	: input/output that will contains all the detection candidates.
dnnRes	raw results of the vpDetectorDNNOpenCV::detect step.
netConfig	the configuration of the network, to know for instance the DNN input size.

cx,cy,w,h,box_score,class_score

cx

cy

w

h

Definition at line 818 of file vpDetectorDNNOpenCV.cpp.

References vpDetectorDNNOpenCV::DetectionCandidates::m_boxes, vpDetectorDNNOpenCV::DetectionCandidates::m_classIds, vpDetectorDNNOpenCV::DetectionCandidates::m_confidences, and m_img.

Referenced by postProcess().

◆ readNet()

void vpDetectorDNNOpenCV::readNet	(	const std::string &	model,
		const std::string &	config = `""`,
		const std::string &	framework = `""`
	)

Read a network, see OpenCV readNet documentation for more information.

Parameters

model	Path to a binary file of model containing trained weights. The following file extensions are expected for models from different frameworks: `.caffemodel` (Caffe, http://caffe.berkeleyvision.org/) `.pb` (TensorFlow, https://www.tensorflow.org/) `.t7` \| `.net` (Torch, http://torch.ch/) `.weights` (Darknet, https://pjreddie.com/darknet/) `.bin` (DLDT, https://software.intel.com/openvino-toolkit) `*.onnx` (ONNX, https://onnx.ai/)
config	Path to a text file of model containing network configuration. It could be a file with the following extensions: `.prototxt` (Caffe, http://caffe.berkeleyvision.org/) `.pbtxt` (TensorFlow, https://www.tensorflow.org/) `.cfg` (Darknet, https://pjreddie.com/darknet/) `.xml` (DLDT, https://software.intel.com/openvino-toolkit)
framework	Optional name of an origin framework of the model. Automatically detected if it is not set.

Definition at line 1044 of file vpDetectorDNNOpenCV.cpp.

References getOutputsNames(), m_net, m_netConfig, and m_outNames.

Referenced by initFromJSON(), setNetConfig(), and vpDetectorDNNOpenCV().

◆ saveConfigurationInJSON()

void vpDetectorDNNOpenCV::saveConfigurationInJSON ( const std::string & jsonPath ) const

Save the network configuration in a JSON file.

Parameters

jsonPath Path towards the output JSON file .

Examples: tutorial-dnn-object-detection-live.cpp.

Definition at line 228 of file vpDetectorDNNOpenCV.cpp.

◆ setConfidenceThreshold()

void vpDetectorDNNOpenCV::setConfidenceThreshold ( const float & confThreshold )

Set confidence threshold to filter the detections.

Parameters

confThreshold Confidence threshold between [0, 1]

Definition at line 1078 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

◆ setDetectionFilterSizeRatio()

void vpDetectorDNNOpenCV::setDetectionFilterSizeRatio ( const double & sizeRatio )

Set the size ratio used in the filterDetection method. If <= 0., filterDetection is not used. The detections of class c for which the bbox area does not belong to [mean_class_c(Area) * sizeRatio ; mean_class_c(Area) / sizeRatio ] are removed from the list of detections.

Parameters

sizeRatio the size ratio used in the filterDetection method. If <= 0., filterDetection is not used.

Definition at line 1095 of file vpDetectorDNNOpenCV.cpp.

References m_applySizeFilterAfterNMS, and m_netConfig.

Referenced by setNetConfig(), and vpDetectorDNNOpenCV().

◆ setInputSize()

void vpDetectorDNNOpenCV::setInputSize	(	const int &	width,
		const int &	height
	)

Set dimension to resize the image to the input blob.

Parameters

width	If <= 0, blob width is set to image width
height	If <= 0, blob height is set to image height

Definition at line 1112 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

◆ setMean()

void vpDetectorDNNOpenCV::setMean	(	const double &	meanR,
		const double &	meanG,
		const double &	meanB
	)

Set mean subtraction values.

Parameters

meanR	Mean value for R-channel
meanG	Mean value for G-channel
meanB	Mean value for R-channel

Definition at line 1125 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

◆ setNetConfig()

void vpDetectorDNNOpenCV::setNetConfig ( const NetConfig & config )

Configure the DNN (thresholds, input size, ...). If the DNN weights file is known, initialize the m_net by reading the weights.

Parameters

config the desired configuration of the network

Examples: tutorial-dnn-object-detection-live.cpp.

Definition at line 1063 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig, readNet(), setDetectionFilterSizeRatio(), and setParsingMethod().

◆ setNMSThreshold()

void vpDetectorDNNOpenCV::setNMSThreshold ( const float & nmsThreshold )

Set Non-Maximum Suppression threshold, used to filter multiple detections at approximatively the same location.

Parameters

nmsThreshold Non-Maximum Suppression threshold between [0, 1]

Definition at line 1086 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

◆ setParsingMethod()

void vpDetectorDNNOpenCV::setParsingMethod	(	const DNNResultsParsingType &	typeParsingMethod,
		void(*)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)	parsingMethod = `postProcess_unimplemented`
	)

Set the type of parsing method that must be used to interpret the raw results of the DNN detection.

Parameters

typeParsingMethod	the type of parsing method that must be used to interpret the raw results of the DNN detection.
parsingMethod	if typeParsingMethod is equal to vpDetectorDNNOpenCV::USER_DEFINED , a function permitting to interpret the cv::Mat resulting from the DNN inference.

Definition at line 1168 of file vpDetectorDNNOpenCV.cpp.

References dnnResultsParsingTypeToString(), m_netConfig, m_parsingMethod, SSD_MOBILENET, YOLO_V7, and YOLO_V8.

Referenced by setNetConfig(), and vpDetectorDNNOpenCV().

◆ setPreferableBackend()

void vpDetectorDNNOpenCV::setPreferableBackend ( const int & backendId )

Set preferable backend for inference computation. See OpenCV setPreferableBackend documentation for more information.

Parameters

backendId Backend identifier

Definition at line 1133 of file vpDetectorDNNOpenCV.cpp.

References m_net.

◆ setPreferableTarget()

void vpDetectorDNNOpenCV::setPreferableTarget ( const int & targetId )

Set preferable target for inference computation. See OpenCV setPreferableTarget documentation for more information.

Parameters

targetId Target identifier

Definition at line 1141 of file vpDetectorDNNOpenCV.cpp.

References m_net.

◆ setScaleFactor()

void vpDetectorDNNOpenCV::setScaleFactor ( const double & scaleFactor )

Set scale factor to normalize the range of pixel values.

Definition at line 1146 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig, YOLO_V7, and YOLO_V8.

◆ setSwapRB()

void vpDetectorDNNOpenCV::setSwapRB ( const bool & swapRB )

If true, swap R and B channel for mean subtraction. For instance when the network has been trained on RGB image format (OpenCV uses BGR convention).

Definition at line 1159 of file vpDetectorDNNOpenCV.cpp.

References m_netConfig.

Friends And Related Function Documentation

◆ from_json

void from_json	(	const nlohmann::json &	j,
		vpDetectorDNNOpenCV &	network
	)

friend

Read the network configuration from JSON. All values are optional and if an argument is not present, the default value defined in the constructor is kept.

Parameters

j	The JSON object, resulting from the parsing of a JSON file.
network	The network, that will be initialized from the JSON data.

Definition at line 519 of file vpDetectorDNNOpenCV.h.

◆ operator<<

std::ostream& operator<<	(	std::ostream &	os,
		const vpDetectorDNNOpenCV &	network
	)

friend

Definition at line 538 of file vpDetectorDNNOpenCV.h.

◆ to_json

void to_json	(	nlohmann::json &	j,
		const vpDetectorDNNOpenCV &	network
	)

friend

Parse the network configuration into JSON format.

Parameters

j	The JSON parser.
network	The network we want to parse the configuration.

Definition at line 530 of file vpDetectorDNNOpenCV.h.

Member Data Documentation

◆ m_applySizeFilterAfterNMS

bool vpDetectorDNNOpenCV::m_applySizeFilterAfterNMS

protected

If true, filter the detections removing the ones for which the bbox does not respect area(bbox) € [mean_class(area) * ratio; mean_class(area) / ratio].

Definition at line 576 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), and setDetectionFilterSizeRatio().

◆ m_blob

cv::Mat vpDetectorDNNOpenCV::m_blob

protected

Buffer for the blob in input net.

Definition at line 578 of file vpDetectorDNNOpenCV.h.

Referenced by detect().

◆ m_dnnRes

std::vector<cv::Mat> vpDetectorDNNOpenCV::m_dnnRes

protected

Contains all output blobs for each layer specified in m_outNames.

Definition at line 592 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), and postProcess().

◆ m_I_color

vpImage<vpRGBa> vpDetectorDNNOpenCV::m_I_color

protected

Buffer for gray to RGBa image conversion.

Definition at line 580 of file vpDetectorDNNOpenCV.h.

Referenced by detect().

◆ m_img

cv::Mat vpDetectorDNNOpenCV::m_img

protected

Buffer for the input image.

Definition at line 582 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), postProcess_FasterRCNN(), postProcess_ResNet_10(), postProcess_SSD_MobileNet(), postProcess_YoloV3_V4(), postProcess_YoloV5_V7(), and postProcess_YoloV8().

◆ m_indices

std::vector<int> vpDetectorDNNOpenCV::m_indices

protected

Indices for NMS.

Definition at line 584 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), and postProcess().

◆ m_net

cv::dnn::Net vpDetectorDNNOpenCV::m_net

protected

DNN network.

Definition at line 586 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), getOutputsNames(), readNet(), setPreferableBackend(), and setPreferableTarget().

◆ m_netConfig

NetConfig vpDetectorDNNOpenCV::m_netConfig

protected

Configuration of the DNN.

Definition at line 588 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), initFromJSON(), postProcess(), readNet(), setConfidenceThreshold(), setDetectionFilterSizeRatio(), setInputSize(), setMean(), setNetConfig(), setNMSThreshold(), setParsingMethod(), setScaleFactor(), setSwapRB(), and vpDetectorDNNOpenCV().

◆ m_outNames

std::vector<cv::String> vpDetectorDNNOpenCV::m_outNames

protected

Names of layers with unconnected outputs.

Definition at line 590 of file vpDetectorDNNOpenCV.h.

Referenced by detect(), postProcess_SSD_MobileNet(), and readNet().

◆ m_parsingMethod

void(* vpDetectorDNNOpenCV::m_parsingMethod) (DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)

protected

Pointer towards the parsing method, used if m_parsingMethodType is equal to m_parsingMethodType::USER_SPECIFIED.

Definition at line 594 of file vpDetectorDNNOpenCV.h.

Referenced by postProcess(), and setParsingMethod().

Classes

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Friends

Detailed Description

Member Typedef Documentation

◆ DetectedFeatures2D

◆ DetectionCandidates

◆ DNNResultsParsingType

◆ NetConfig

Member Enumeration Documentation

◆ DNNResultsParsingType

Constructor & Destructor Documentation

◆ vpDetectorDNNOpenCV() [1/3]

◆ vpDetectorDNNOpenCV() [2/3]

◆ vpDetectorDNNOpenCV() [3/3]

◆ ~vpDetectorDNNOpenCV()

Member Function Documentation

◆ detect() [1/9]

◆ detect() [2/9]

◆ detect() [3/9]

◆ detect() [4/9]

◆ detect() [5/9]

◆ detect() [6/9]

◆ detect() [7/9]

◆ detect() [8/9]

◆ detect() [9/9]

◆ dnnResultsParsingTypeFromString()

◆ dnnResultsParsingTypeToString()

◆ filterDetectionMultiClassInput() [1/2]

◆ filterDetectionMultiClassInput() [2/2]

◆ filterDetectionSingleClassInput()

◆ getAvailableDnnResultsParsingTypes()

◆ getNetConfig()

◆ getOutputsNames()

◆ initFromJSON()

◆ parseClassNamesFile()

◆ postProcess()

◆ postProcess_FasterRCNN()

◆ postProcess_ResNet_10()

◆ postProcess_SSD_MobileNet()

◆ postProcess_unimplemented()

◆ postProcess_YoloV3_V4()

◆ postProcess_YoloV5_V7()

◆ postProcess_YoloV8()

◆ readNet()

◆ saveConfigurationInJSON()

◆ setConfidenceThreshold()

◆ setDetectionFilterSizeRatio()

◆ setInputSize()

◆ setMean()

◆ setNetConfig()

◆ setNMSThreshold()

◆ setParsingMethod()

◆ setPreferableBackend()

◆ setPreferableTarget()

◆ setScaleFactor()

◆ setSwapRB()

Friends And Related Function Documentation

◆ from_json

◆ operator<<

◆ to_json

Member Data Documentation

◆ m_applySizeFilterAfterNMS

◆ m_blob

◆ m_dnnRes

◆ m_I_color

◆ m_img

◆ m_indices

◆ m_net

◆ m_netConfig

◆ m_outNames

◆ m_parsingMethod