OpenShot Library | libopenshot  0.2.6
CVObjectDetection.h
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Header file for CVObjectDetection class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  * @author Brenno Caldato <brenno.caldato@outlook.com>
6  *
7  * @ref License
8  */
9 
10 /* LICENSE
11  *
12  * Copyright (c) 2008-2019 OpenShot Studios, LLC
13  * <http://www.openshotstudios.com/>. This file is part of
14  * OpenShot Library (libopenshot), an open-source project dedicated to
15  * delivering high quality video editing and animation solutions to the
16  * world. For more information visit <http://www.openshot.org/>.
17  *
18  * OpenShot Library (libopenshot) is free software: you can redistribute it
19  * and/or modify it under the terms of the GNU Lesser General Public License
20  * as published by the Free Software Foundation, either version 3 of the
21  * License, or (at your option) any later version.
22  *
23  * OpenShot Library (libopenshot) is distributed in the hope that it will be
24  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU Lesser General Public License for more details.
27  *
28  * You should have received a copy of the GNU Lesser General Public License
29  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30  */
31 
32 #pragma once
33 
34 #define int64 opencv_broken_int
35 #define uint64 opencv_broken_uint
36 #include <opencv2/dnn.hpp>
37 #include <opencv2/opencv.hpp>
38 #include <opencv2/core.hpp>
39 #undef uint64
40 #undef int64
41 #include "Json.h"
42 #include "ProcessingController.h"
43 #include "Clip.h"
44 #include "protobuf_messages/objdetectdata.pb.h"
45 
46 #include "sort_filter/sort.hpp"
47 
48 namespace openshot
49 {
50  // Stores the detected object bounding boxes and its properties.
54  std::vector<int> _classIds,
55  std::vector<float> _confidences,
56  std::vector<cv::Rect_<float>> _boxes,
57  size_t _frameId,
58  std::vector<int> _objectIds)
59  {
60  classIds = _classIds;
61  confidences = _confidences;
62  boxes = _boxes;
63  frameId = _frameId;
64  objectIds = _objectIds;
65  }
66  size_t frameId;
67  std::vector<int> classIds;
68  std::vector<float> confidences;
69  std::vector<cv::Rect_<float>> boxes;
70  std::vector<int> objectIds;
71  };
72 
73  /**
74  * @brief This class runs trought a clip to detect objects and returns the bounding boxes and its properties.
75  *
76  * Object detection is performed using YoloV3 model with OpenCV DNN module
77  */
79 
80  private:
81 
82  cv::dnn::Net net;
83  std::vector<std::string> classNames;
84  float confThreshold, nmsThreshold;
85 
86  std::string classesFile;
87  std::string modelConfiguration;
88  std::string modelWeights;
89  std::string processingDevice;
90  std::string protobuf_data_path;
91 
92  SortTracker sort;
93 
94  uint progress;
95 
96  size_t start;
97  size_t end;
98 
99  bool error = false;
100 
101  /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes
102  ProcessingController *processingController;
103 
104  void setProcessingDevice();
105 
106  // Detect onbects on a single frame
107  void DetectObjects(const cv::Mat &frame, size_t frame_number);
108 
109  bool iou(cv::Rect pred_box, cv::Rect sort_box);
110 
111  // Remove the bounding boxes with low confidence using non-maxima suppression
112  void postprocess(const cv::Size &frameDims, const std::vector<cv::Mat>& out, size_t frame_number);
113 
114  // Get the names of the output layers
115  std::vector<cv::String> getOutputsNames(const cv::dnn::Net& net);
116 
117  public:
118 
119  std::map<size_t, CVDetectionData> detectionsData;
120 
121  CVObjectDetection(std::string processInfoJson, ProcessingController &processingController);
122 
123  // Iterate over a clip object and run inference for each video frame
124  void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false);
125 
126  CVDetectionData GetDetectionData(size_t frameId);
127 
128  /// Protobuf Save and Load methods
129  // Save protobuf file
130  bool SaveObjDetectedData();
131  // Add frame object detection data into protobuf message.
132  void AddFrameDataToProto(pb_objdetect::Frame* pbFrameData, CVDetectionData& dData);
133 
134  // Get and Set JSON methods
135  void SetJson(const std::string value); ///< Load JSON string into this object
136  void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
137 
138  // Load protobuf file (ONLY FOR MAKE TEST)
139  bool _LoadObjDetectdData();
140  };
141 
142 }
std::vector< int > objectIds
std::vector< cv::Rect_< float > > boxes
This is a message class for thread safe comunication between ClipProcessingJobs and OpenCV classes...
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
This class runs trought a clip to detect objects and returns the bounding boxes and its properties...
Header file for Clip class.
Header file for JSON class.
std::vector< int > classIds
std::map< size_t, CVDetectionData > detectionsData
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:46
std::vector< float > confidences
CVDetectionData(std::vector< int > _classIds, std::vector< float > _confidences, std::vector< cv::Rect_< float >> _boxes, size_t _frameId, std::vector< int > _objectIds)