OpenShot Library | libopenshot  0.2.6
Tracker.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for Tracker effect class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  * @author Brenno Caldato <brenno.caldato@outlook.com>
6  *
7  * @ref License
8  */
9 
10 /* LICENSE
11  *
12  * Copyright (c) 2008-2019 OpenShot Studios, LLC
13  * <http://www.openshotstudios.com/>. This file is part of
14  * OpenShot Library (libopenshot), an open-source project dedicated to
15  * delivering high quality video editing and animation solutions to the
16  * world. For more information visit <http://www.openshot.org/>.
17  *
18  * OpenShot Library (libopenshot) is free software: you can redistribute it
19  * and/or modify it under the terms of the GNU Lesser General Public License
20  * as published by the Free Software Foundation, either version 3 of the
21  * License, or (at your option) any later version.
22  *
23  * OpenShot Library (libopenshot) is distributed in the hope that it will be
24  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU Lesser General Public License for more details.
27  *
28  * You should have received a copy of the GNU Lesser General Public License
29  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30  */
31 
32 #include <string>
33 #include <memory>
34 #include <fstream>
35 #include <iostream>
36 
37 #include "effects/Tracker.h"
38 #include "Exceptions.h"
39 #include "Timeline.h"
40 
41 #include <google/protobuf/util/time_util.h>
42 
43 #include <QImage>
44 #include <QPainter>
45 #include <QRectF>
46 
47 using namespace std;
48 using namespace openshot;
49 using google::protobuf::util::TimeUtil;
50 
51 /// Blank constructor, useful when using Json to load the effect properties
52 Tracker::Tracker(std::string clipTrackerDataPath)
53 {
54  // Init effect properties
55  init_effect_details();
56  // Instantiate a TrackedObjectBBox object and point to it
57  TrackedObjectBBox trackedDataObject;
58  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
59  // Tries to load the tracked object's data from protobuf file
60  trackedData->LoadBoxData(clipTrackerDataPath);
61  ClipBase* parentClip = this->ParentClip();
62  trackedData->ParentClip(parentClip);
63  trackedData->Id(std::to_string(0));
64  // Insert TrackedObject with index 0 to the trackedObjects map
65  trackedObjects.insert({0, trackedData});
66 }
67 
68 // Default constructor
69 Tracker::Tracker()
70 {
71  // Init effect properties
72  init_effect_details();
73  // Instantiate a TrackedObjectBBox object and point to it
74  TrackedObjectBBox trackedDataObject;
75  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
76  ClipBase* parentClip = this->ParentClip();
77  trackedData->ParentClip(parentClip);
78  trackedData->Id(std::to_string(0));
79  // Insert TrackedObject with index 0 to the trackedObjects map
80  trackedObjects.insert({0, trackedData});
81 }
82 
83 
84 // Init effect settings
85 void Tracker::init_effect_details()
86 {
87  /// Initialize the values of the EffectInfo struct.
88  InitEffectInfo();
89 
90  /// Set the effect info
91  info.class_name = "Tracker";
92  info.name = "Tracker";
93  info.description = "Track the selected bounding box through the video.";
94  info.has_audio = false;
95  info.has_video = true;
96  info.has_tracked_object = true;
97 
98  this->TimeScale = 1.0;
99 }
100 
101 // This method is required for all derived classes of EffectBase, and returns a
102 // modified openshot::Frame object
103 std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
104 {
105  // Get the frame's image
106  cv::Mat frame_image = frame->GetImageCV();
107 
108  // Initialize the Qt rectangle that will hold the positions of the bounding-box
109  QRectF boxRect;
110  // Initialize the image of the TrackedObject child clip
111  std::shared_ptr<QImage> childClipImage = nullptr;
112 
113  // Check if frame isn't NULL
114  if(!frame_image.empty() &&
115  trackedData->Contains(frame_number) &&
116  trackedData->visible.GetValue(frame_number) == 1)
117  {
118  // Get the width and height of the image
119  float fw = frame_image.size().width;
120  float fh = frame_image.size().height;
121 
122  // Get the bounding-box of given frame
123  BBox fd = trackedData->GetBox(frame_number);
124 
125  // Check if track data exists for the requested frame
126  if (trackedData->draw_box.GetValue(frame_number) == 1)
127  {
128  std::vector<int> stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number);
129  int stroke_width = trackedData->stroke_width.GetValue(frame_number);
130  float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number);
131  std::vector<int> bg_rgba = trackedData->background.GetColorRGBA(frame_number);
132  float bg_alpha = trackedData->background_alpha.GetValue(frame_number);
133 
134  // Create a rotated rectangle object that holds the bounding box
135  cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
136  cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
137  (int) (fd.angle) );
138 
139  DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true);
140  DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false);
141  }
142 
143  // Get the image of the Tracked Object' child clip
144  if (trackedData->ChildClipId() != ""){
145  // Cast the parent timeline of this effect
146  Timeline* parentTimeline = (Timeline *) ParentTimeline();
147  if (parentTimeline){
148  // Get the Tracked Object's child clip
149  Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId());
150  if (childClip){
151  // Get the image of the child clip for this frame
152  std::shared_ptr<Frame> f(new Frame(1, frame->GetWidth(), frame->GetHeight(), "#00000000"));
153  std::shared_ptr<Frame> childClipFrame = childClip->GetFrame(f, frame_number);
154  childClipImage = childClipFrame->GetImage();
155 
156  // Set the Qt rectangle with the bounding-box properties
157  boxRect.setRect((int)((fd.cx-fd.width/2)*fw),
158  (int)((fd.cy - fd.height/2)*fh),
159  (int)(fd.width*fw),
160  (int)(fd.height*fh) );
161  }
162  }
163  }
164 
165  }
166 
167  // Set image with drawn box to frame
168  // If the input image is NULL or doesn't have tracking data, it's returned as it came
169  frame->SetImageCV(frame_image);
170 
171  // Set the bounding-box image with the Tracked Object's child clip image
172  if (childClipImage){
173  // Get the frame image
174  QImage frameImage = *(frame->GetImage());
175 
176  // Set a Qt painter to the frame image
177  QPainter painter(&frameImage);
178 
179  // Draw the child clip image inside the bounding-box
180  painter.drawImage(boxRect, *childClipImage, QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
181 
182  // Set the frame image as the composed image
183  frame->AddImage(std::make_shared<QImage>(frameImage));
184  }
185 
186  return frame;
187 }
188 
189 void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color, float alpha, int thickness, bool is_background){
190  // Get the bouding box vertices
191  cv::Point2f vertices2f[4];
192  box.points(vertices2f);
193 
194  // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed
195  // select min enclosing rectangle to draw on a small portion of the image
196  // cv::Rect rect = box.boundingRect();
197  // cv::Mat image = frame_image(rect)
198 
199  if(is_background){
200  cv::Mat overlayFrame;
201  frame_image.copyTo(overlayFrame);
202 
203  // draw bounding box background
204  cv::Point vertices[4];
205  for(int i = 0; i < 4; ++i){
206  vertices[i] = vertices2f[i];}
207 
208  cv::Rect rect = box.boundingRect();
209  cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
210  // add opacity
211  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
212  }
213  else{
214  cv::Mat overlayFrame;
215  frame_image.copyTo(overlayFrame);
216 
217  // Draw bounding box
218  for (int i = 0; i < 4; i++)
219  {
220  cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
221  thickness, cv::LINE_AA);
222  }
223 
224  // add opacity
225  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
226  }
227 }
228 
229 // Get the indexes and IDs of all visible objects in the given frame
230 std::string Tracker::GetVisibleObjects(int64_t frame_number) const{
231 
232  // Initialize the JSON objects
233  Json::Value root;
234  root["visible_objects_index"] = Json::Value(Json::arrayValue);
235  root["visible_objects_id"] = Json::Value(Json::arrayValue);
236 
237  // Iterate through the tracked objects
238  for (const auto& trackedObject : trackedObjects){
239  // Get the tracked object JSON properties for this frame
240  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
241  if (trackedObjectJSON["visible"]["value"].asBool()){
242  // Save the object's index and ID if it's visible in this frame
243  root["visible_objects_index"].append(trackedObject.first);
244  root["visible_objects_id"].append(trackedObject.second->Id());
245  }
246  }
247 
248  return root.toStyledString();
249 }
250 
251 // Generate JSON string of this object
252 std::string Tracker::Json() const {
253 
254  // Return formatted string
255  return JsonValue().toStyledString();
256 }
257 
258 // Generate Json::Value for this object
259 Json::Value Tracker::JsonValue() const {
260 
261  // Create root json object
262  Json::Value root = EffectBase::JsonValue(); // get parent properties
263 
264  // Save the effect's properties on root
265  root["type"] = info.class_name;
266  root["protobuf_data_path"] = protobuf_data_path;
267  root["BaseFPS"]["num"] = BaseFPS.num;
268  root["BaseFPS"]["den"] = BaseFPS.den;
269  root["TimeScale"] = this->TimeScale;
270 
271  // Add trackedObjects IDs to JSON
272  Json::Value objects;
273  for (auto const& trackedObject : trackedObjects){
274  Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
275  // add object json
276  objects[trackedObject.second->Id()] = trackedObjectJSON;
277  }
278  root["objects"] = objects;
279 
280  // return JsonValue
281  return root;
282 }
283 
284 // Load JSON string into this object
285 void Tracker::SetJson(const std::string value) {
286 
287  // Parse JSON string into JSON objects
288  try
289  {
290  const Json::Value root = openshot::stringToJson(value);
291  // Set all values that match
292  SetJsonValue(root);
293  }
294  catch (const std::exception& e)
295  {
296  // Error parsing JSON (or missing keys)
297  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
298  }
299  return;
300 }
301 
302 // Load Json::Value into this object
303 void Tracker::SetJsonValue(const Json::Value root) {
304 
305  // Set parent data
306  EffectBase::SetJsonValue(root);
307 
308  if(!root["type"].isNull())
309  info.class_name = root["type"].asString();
310 
311  if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
312  {
313  if (!root["BaseFPS"]["num"].isNull())
314  {
315  BaseFPS.num = (int) root["BaseFPS"]["num"].asInt();
316  }
317  if (!root["BaseFPS"]["den"].isNull())
318  {
319  BaseFPS.den = (int) root["BaseFPS"]["den"].asInt();
320  }
321  }
322 
323  if (!root["TimeScale"].isNull())
324  TimeScale = (double) root["TimeScale"].asDouble();
325 
326  // Set data from Json (if key is found)
327  if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1)
328  {
329  protobuf_data_path = root["protobuf_data_path"].asString();
330  if(!trackedData->LoadBoxData(protobuf_data_path))
331  {
332  std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n';
333  protobuf_data_path = "";
334  }
335  }
336 
337  if (!root["objects"].isNull()){
338  for (auto const& trackedObject : trackedObjects){
339  std::string obj_id = std::to_string(trackedObject.first);
340  if(!root["objects"][obj_id].isNull()){
341  trackedObject.second->SetJsonValue(root["objects"][obj_id]);
342  }
343  }
344  }
345 
346  // Set the tracked object's ids
347  if (!root["objects_id"].isNull()){
348  for (auto const& trackedObject : trackedObjects){
349  Json::Value trackedObjectJSON;
350  trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
351  trackedObject.second->SetJsonValue(trackedObjectJSON);
352  }
353  }
354 
355  return;
356 }
357 
358 // Get all properties for a specific frame
359 std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
360 
361  // Generate JSON properties list
362  Json::Value root;
363 
364  // Add trackedObject properties to JSON
365  Json::Value objects;
366  for (auto const& trackedObject : trackedObjects){
367  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
368  // add object json
369  objects[trackedObject.second->Id()] = trackedObjectJSON;
370  }
371  root["objects"] = objects;
372 
373  // Append effect's properties
374  root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
375  root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
376  root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
377  root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
378  root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
379  root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
380 
381  // Return formatted string
382  return root.toStyledString();
383 }
Header file for Tracker effect class.
float cy
y-coordinate of the bounding box center
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:107
float height
bounding box height
STL namespace.
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:404
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
float angle
bounding box rotation angle [degrees]
Header file for Timeline class.
Header file for all Exception classes.
bool LoadBoxData(std::string inputFilePath)
Load the bounding-boxes information from the protobuf file.
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
float width
bounding box width
This abstract class is the base class, used by all clips in libopenshot.
Definition: ClipBase.h:51
This struct holds the information of a bounding-box.
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:46
float cx
x-coordinate of the bounding box center
Exception for invalid JSON.
Definition: Exceptions.h:205
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:360
This class contains the properties of a tracked object and functions to manipulate it...
This class represents a timeline.
Definition: Timeline.h:168