35 #include <visp3/core/vpConfig.h> 37 #if (VISP_HAVE_OPENCV_VERSION >= 0x030403) 38 #include <visp3/detection/vpDetectorDNN.h> 39 #include <visp3/core/vpImageConvert.h> 42 m_confidenceThreshold(0.5), m_I_color(), m_img(), m_inputSize(300,300), m_mean(127.5, 127.5, 127.5),
43 m_net(), m_nmsThreshold(0.4f), m_outNames(), m_outs(), m_scaleFactor(2.0/255.0), m_swapRB(true) {}
58 std::vector<vpRect> boundingBoxes;
59 return detect(m_I_color, boundingBoxes);
72 cv::Size inputSize(m_inputSize.width > 0 ? m_inputSize.width : m_img.cols,
73 m_inputSize.height > 0 ? m_inputSize.height : m_img.rows);
74 cv::dnn::blobFromImage(m_img, m_blob, m_scaleFactor, inputSize, m_mean, m_swapRB,
false);
76 m_net.setInput(m_blob);
77 m_net.forward(m_outs, m_outNames);
81 boundingBoxes.resize(m_boxesNMS.size());
82 for (
size_t i = 0; i < m_boxesNMS.size(); i++) {
83 cv::Rect box = m_boxesNMS[i];
84 boundingBoxes[i] =
vpRect(box.x, box.y, box.width, box.height);
90 for (
size_t i = 0; i < boundingBoxes.size(); i++) {
91 std::vector<vpImagePoint> polygon;
93 double x = boundingBoxes[i].getLeft();
94 double y = boundingBoxes[i].getTop();
95 double w = boundingBoxes[i].getWidth();
96 double h = boundingBoxes[i].getHeight();
105 std::ostringstream oss;
106 oss << m_classIds[i] <<
" ; " << m_confidences[i] <<
" ; " << m_boxes[i];
110 return !boundingBoxes.empty();
119 std::vector<vpRect> bbs;
121 bbs.reserve(m_boxesNMS.size());
122 for (
size_t i = 0; i < m_boxesNMS.size(); i++) {
123 cv::Rect box = m_boxes[i];
124 bbs.push_back(
vpRect(box.x, box.y, box.width, box.height));
127 bbs.reserve(m_boxes.size());
128 for (
size_t i = 0; i < m_boxes.size(); i++) {
129 cv::Rect box = m_boxes[i];
130 bbs.push_back(
vpRect(box.x, box.y, box.width, box.height));
144 std::vector<int> classIds;
145 for (
size_t i = 0; i < m_indices.size(); i++) {
146 int idx = m_indices[i];
147 classIds.push_back(m_classIds[idx]);
160 std::vector<float> confidences;
161 for (
size_t i = 0; i < m_indices.size(); i++) {
162 int idx = m_indices[i];
163 confidences.push_back(m_confidences[idx]);
168 return m_confidences;
171 #if (VISP_HAVE_OPENCV_VERSION == 0x030403) 172 std::vector<cv::String> vpDetectorDNN::getOutputsNames() {
173 static std::vector<cv::String> names;
175 std::vector<int> outLayers = m_net.getUnconnectedOutLayers();
176 std::vector<cv::String> layersNames = m_net.getLayerNames();
177 names.resize(outLayers.size());
178 for (
size_t i = 0; i < outLayers.size(); ++i)
179 names[i] = layersNames[outLayers[i] - 1];
185 void vpDetectorDNN::postProcess() {
187 static std::vector<int> outLayers = m_net.getUnconnectedOutLayers();
188 static std::string outLayerType = m_net.getLayer(outLayers[0])->type;
191 m_confidences.clear();
193 if (m_net.getLayer(0)->outputNameToIndex(
"im_info") != -1)
198 CV_Assert(m_outs.size() == 1);
199 float* data = (
float*)m_outs[0].data;
200 for (
size_t i = 0; i < m_outs[0].total(); i += 7)
202 float confidence = data[i + 2];
203 if (confidence > m_confidenceThreshold)
205 int left = (int)data[i + 3];
206 int top = (int)data[i + 4];
207 int right = (int)data[i + 5];
208 int bottom = (int)data[i + 6];
209 int width = right - left + 1;
210 int height = bottom - top + 1;
211 m_classIds.push_back((
int)(data[i + 1]) - 1);
212 m_boxes.push_back(cv::Rect(left, top, width, height));
213 m_confidences.push_back(confidence);
217 else if (outLayerType ==
"DetectionOutput")
222 CV_Assert(m_outs.size() == 1);
223 float* data = (
float*)m_outs[0].data;
224 for (
size_t i = 0; i < m_outs[0].total(); i += 7)
226 float confidence = data[i + 2];
227 if (confidence > m_confidenceThreshold)
229 int left = (int)(data[i + 3] * m_img.cols);
230 int top = (int)(data[i + 4] * m_img.rows);
231 int right = (int)(data[i + 5] * m_img.cols);
232 int bottom = (int)(data[i + 6] * m_img.rows);
233 int width = right - left + 1;
234 int height = bottom - top + 1;
235 m_classIds.push_back((
int)(data[i + 1]) - 1);
236 m_boxes.push_back(cv::Rect(left, top, width, height));
237 m_confidences.push_back(confidence);
241 else if (outLayerType ==
"Region")
243 for (
size_t i = 0; i < m_outs.size(); ++i)
248 float* data = (
float*)m_outs[i].data;
249 for (
int j = 0; j < m_outs[i].rows; ++j, data += m_outs[i].cols)
251 cv::Mat scores = m_outs[i].row(j).colRange(5, m_outs[i].cols);
252 cv::Point classIdPoint;
254 cv::minMaxLoc(scores, 0, &confidence, 0, &classIdPoint);
255 if (confidence > m_confidenceThreshold)
257 int centerX = (int)(data[0] * m_img.cols);
258 int centerY = (int)(data[1] * m_img.rows);
259 int width = (int)(data[2] * m_img.cols);
260 int height = (int)(data[3] * m_img.rows);
261 int left = centerX - width / 2;
262 int top = centerY - height / 2;
264 m_classIds.push_back(classIdPoint.x);
265 m_confidences.push_back((
float)confidence);
266 m_boxes.push_back(cv::Rect(left, top, width, height));
272 CV_Error(cv::Error::StsNotImplemented,
"Unknown output layer type: " + outLayerType);
274 cv::dnn::NMSBoxes(m_boxes, m_confidences, m_confidenceThreshold, m_nmsThreshold, m_indices);
275 m_boxesNMS.resize(m_indices.size());
276 for (
size_t i = 0; i < m_indices.size(); ++i)
278 int idx = m_indices[i];
279 m_boxesNMS[i] = m_boxes[idx];
302 m_net = cv::dnn::readNet(model, config, framework);
303 #if (VISP_HAVE_OPENCV_VERSION == 0x030403) 304 m_outNames = getOutputsNames();
306 m_outNames = m_net.getUnconnectedOutLayersNames();
316 m_confidenceThreshold = confThreshold;
326 m_inputSize.width = width;
327 m_inputSize.height = height;
338 m_mean = cv::Scalar(meanR, meanG, meanB);
348 m_nmsThreshold = nmsThreshold;
358 m_net.setPreferableBackend(backendId);
368 m_net.setPreferableTarget(targetId);
375 m_scaleFactor = scaleFactor;
387 #elif !defined(VISP_BUILD_SHARED_LIBS) 390 void dummy_vpDetectorDNN(){};
void setMean(double meanR, double meanG, double meanB)
static void convert(const vpImage< unsigned char > &src, vpImage< vpRGBa > &dest)
void setPreferableTarget(int targetId)
std::vector< int > getDetectionClassIds(bool afterNMS=true) const
void setPreferableBackend(int backendId)
void readNet(const std::string &model, const std::string &config="", const std::string &framework="")
std::vector< vpRect > getDetectionBBs(bool afterNMS=true) const
size_t m_nb_objects
Number of detected objects.
void setSwapRB(bool swapRB)
void setScaleFactor(double scaleFactor)
void setNMSThreshold(float nmsThreshold)
void setInputSize(int width, int height)
void setConfidenceThreshold(float confThreshold)
std::vector< std::vector< vpImagePoint > > m_polygon
Defines a rectangle in the plane.
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
std::vector< std::string > m_message
Message attached to each object.
virtual bool detect(const vpImage< unsigned char > &I)
std::vector< float > getDetectionConfidence(bool afterNMS=true) const