OpenShot Library | libopenshot  0.2.6
FFmpegReader.h
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Header file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12  * (http://www.openshotstudios.com). This file is part of
13  * OpenShot Library (http://www.openshot.org), an open-source project
14  * dedicated to delivering high quality video editing and animation solutions
15  * to the world.
16  *
17  * This file is originally based on the Libavformat API example, and then modified
18  * by the libopenshot project.
19  *
20  * OpenShot Library (libopenshot) is free software: you can redistribute it
21  * and/or modify it under the terms of the GNU Lesser General Public License
22  * as published by the Free Software Foundation, either version 3 of the
23  * License, or (at your option) any later version.
24  *
25  * OpenShot Library (libopenshot) is distributed in the hope that it will be
26  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28  * GNU Lesser General Public License for more details.
29  *
30  * You should have received a copy of the GNU Lesser General Public License
31  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32  */
33 
34 #ifndef OPENSHOT_FFMPEG_READER_H
35 #define OPENSHOT_FFMPEG_READER_H
36 
37 #include "ReaderBase.h"
38 
39 // Include FFmpeg headers and macros
40 #include "FFmpegUtilities.h"
41 
42 #include <cmath>
43 #include <ctime>
44 #include <iostream>
45 #include <stdio.h>
46 #include <memory>
47 #include "CacheMemory.h"
48 #include "Clip.h"
49 #include "OpenMPUtilities.h"
50 #include "Settings.h"
51 
52 
53 namespace openshot {
54  /**
55  * @brief This struct holds the associated video frame and starting sample # for an audio packet.
56  *
57  * Because audio packets do not match up with video frames, this helps determine exactly
58  * where the audio packet's samples belong.
59  */
60  struct AudioLocation {
61  int64_t frame;
63 
64  bool is_near(AudioLocation location, int samples_per_frame, int64_t amount);
65  };
66 
67  /**
68  * @brief This class uses the FFmpeg libraries, to open video files and audio files, and return
69  * openshot::Frame objects for any frame in the file.
70  *
71  * All seeking and caching is handled internally, and the primary public interface is the GetFrame()
72  * method. To use this reader, simply create an instance of this class, and call the GetFrame method
73  * to start retrieving frames. Use the <b>info</b> struct to obtain information on the file, such as the length
74  * (# of frames), height, width, bit rate, frames per second (fps), etc...
75  *
76  * @code
77  * // Create a reader for a video
78  * openshot::FFmpegReader r("MyAwesomeVideo.webm");
79  * r.Open(); // Open the reader
80  *
81  * // Get frame number 1 from the video
82  * std::shared_ptr<openshot::Frame> f = r.GetFrame(1);
83  *
84  * // Now that we have an openshot::Frame object, lets have some fun!
85  * f->Display(); // Display the frame on the screen
86  * f->DisplayWaveform(); // Display the audio waveform as an image
87  * f->Play(); // Play the audio through your speaker
88  *
89  * // Close the reader
90  * r.Close();
91  * @endcode
92  */
93  class FFmpegReader : public ReaderBase {
94  private:
95  std::string path;
96 
97  AVFormatContext *pFormatCtx;
98  int i, videoStream, audioStream;
99  AVCodecContext *pCodecCtx, *aCodecCtx;
100 #if USE_HW_ACCEL
101  AVBufferRef *hw_device_ctx = NULL; //PM
102 #endif
103  AVStream *pStream, *aStream;
104  AVPacket *packet;
105  AVFrame *pFrame;
106  bool is_open;
107  bool is_duration_known;
108  bool check_interlace;
109  bool check_fps;
110  bool has_missing_frames;
111  int max_concurrent_frames;
112 
113  CacheMemory working_cache;
114  CacheMemory missing_frames;
115  std::map<int64_t, int64_t> processing_video_frames;
116  std::multimap<int64_t, int64_t> processing_audio_frames;
117  std::map<int64_t, int64_t> processed_video_frames;
118  std::map<int64_t, int64_t> processed_audio_frames;
119  std::multimap<int64_t, int64_t> missing_video_frames;
120  std::multimap<int64_t, int64_t> missing_video_frames_source;
121  std::multimap<int64_t, int64_t> missing_audio_frames;
122  std::multimap<int64_t, int64_t> missing_audio_frames_source;
123  std::map<int64_t, int> checked_frames;
124  AudioLocation previous_packet_location;
125 
126  // DEBUG VARIABLES (FOR AUDIO ISSUES)
127  int prev_samples;
128  int64_t prev_pts;
129  int64_t pts_total;
130  int64_t pts_counter;
131  int64_t num_packets_since_video_frame;
132  int64_t num_checks_since_final;
133  std::shared_ptr<openshot::Frame> last_video_frame;
134 
135  bool is_seeking;
136  int64_t seeking_pts;
137  int64_t seeking_frame;
138  bool is_video_seek;
139  int seek_count;
140  int64_t seek_audio_frame_found;
141  int64_t seek_video_frame_found;
142 
143  int64_t audio_pts_offset;
144  int64_t video_pts_offset;
145  int64_t last_frame;
146  int64_t largest_frame_processed;
147  int64_t current_video_frame; // can't reliably use PTS of video to determine this
148 
149  int hw_de_supported = 0; // Is set by FFmpegReader
150 #if USE_HW_ACCEL
151  AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE;
152  AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE;
153  int IsHardwareDecodeSupported(int codecid);
154 #endif
155 
156  /// Check for the correct frames per second value by scanning the 1st few seconds of video packets.
157  void CheckFPS();
158 
159  /// Check the current seek position and determine if we need to seek again
160  bool CheckSeek(bool is_video);
161 
162  /// Check if a frame is missing and attempt to replace its frame image (and
163  bool CheckMissingFrame(int64_t requested_frame);
164 
165  /// Check the working queue, and move finished frames to the finished queue
166  void CheckWorkingFrames(bool end_of_stream, int64_t requested_frame);
167 
168  /// Convert Frame Number into Audio PTS
169  int64_t ConvertFrameToAudioPTS(int64_t frame_number);
170 
171  /// Convert Frame Number into Video PTS
172  int64_t ConvertFrameToVideoPTS(int64_t frame_number);
173 
174  /// Convert Video PTS into Frame Number
175  int64_t ConvertVideoPTStoFrame(int64_t pts);
176 
177  /// Create a new Frame (or return an existing one) and add it to the working queue.
178  std::shared_ptr<openshot::Frame> CreateFrame(int64_t requested_frame);
179 
180  /// Calculate Starting video frame and sample # for an audio PTS
181  AudioLocation GetAudioPTSLocation(int64_t pts);
182 
183  /// Get an AVFrame (if any)
184  bool GetAVFrame();
185 
186  /// Get the next packet (if any)
187  int GetNextPacket();
188 
189  /// Get the smallest video frame that is still being processed
190  int64_t GetSmallestVideoFrame();
191 
192  /// Get the smallest audio frame that is still being processed
193  int64_t GetSmallestAudioFrame();
194 
195  /// Get the PTS for the current video packet
196  int64_t GetVideoPTS();
197 
198  /// Check if there's an album art
199  bool HasAlbumArt();
200 
201  /// Remove partial frames due to seek
202  bool IsPartialFrame(int64_t requested_frame);
203 
204  /// Process a video packet
205  void ProcessVideoPacket(int64_t requested_frame);
206 
207  /// Process an audio packet
208  void ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample);
209 
210  /// Read the stream until we find the requested Frame
211  std::shared_ptr<openshot::Frame> ReadStream(int64_t requested_frame);
212 
213  /// Remove AVFrame from cache (and deallocate its memory)
214  void RemoveAVFrame(AVFrame *);
215 
216  /// Remove AVPacket from cache (and deallocate its memory)
217  void RemoveAVPacket(AVPacket *);
218 
219  /// Seek to a specific Frame. This is not always frame accurate, it's more of an estimation on many codecs.
220  void Seek(int64_t requested_frame);
221 
222  /// Update PTS Offset (if any)
223  void UpdatePTSOffset(bool is_video);
224 
225  /// Update File Info for audio streams
226  void UpdateAudioInfo();
227 
228  /// Update File Info for video streams
229  void UpdateVideoInfo();
230 
231  public:
232  /// Final cache object used to hold final frames
234 
235  /// Enable or disable seeking. Seeking can more quickly locate the requested frame, but some
236  /// codecs have trouble seeking, and can introduce artifacts or blank images into the video.
238 
239  /// @brief Constructor for FFmpegReader.
240  ///
241  /// Sets (and possibly opens) the media file path,
242  /// or throws an exception.
243  /// @param path The filesystem location to load
244  /// @param inspect_reader if true (the default), automatically open the media file and loads frame 1.
245  FFmpegReader(const std::string& path, bool inspect_reader=true);
246 
247  /// Destructor
248  virtual ~FFmpegReader();
249 
250  /// Close File
251  void Close() override;
252 
253  /// Get the cache object used by this reader
254  CacheMemory *GetCache() override { return &final_cache; };
255 
256  /// Get a shared pointer to a openshot::Frame object for a specific frame number of this reader.
257  ///
258  /// @returns The requested frame of video
259  /// @param requested_frame The frame number that is requested.
260  std::shared_ptr<openshot::Frame> GetFrame(int64_t requested_frame) override;
261 
262  /// Determine if reader is open or closed
263  bool IsOpen() override { return is_open; };
264 
265  /// Return the type name of the class
266  std::string Name() override { return "FFmpegReader"; };
267 
268  // Get and Set JSON methods
269  std::string Json() const override; ///< Generate JSON string of this object
270  void SetJson(const std::string value) override; ///< Load JSON string into this object
271  Json::Value JsonValue() const override; ///< Generate Json::Value for this object
272  void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object
273 
274  /// Open File - which is called by the constructor automatically
275  void Open() override;
276 
277  /// Return true if frame can be read with GetFrame()
278  bool GetIsDurationKnown();
279  };
280 
281 }
282 
283 #endif
std::string Name() override
Return the type name of the class.
Definition: FFmpegReader.h:266
Header file for ReaderBase class.
CacheMemory * GetCache() override
Get the cache object used by this reader.
Definition: FFmpegReader.h:254
Header file for OpenMPUtilities (set some common macros)
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:97
Header file for CacheMemory class.
This class uses the FFmpeg libraries, to open video files and audio files, and return openshot::Frame...
Definition: FFmpegReader.h:93
Header file for Clip class.
bool IsOpen() override
Determine if reader is open or closed.
Definition: FFmpegReader.h:263
Header file for global Settings class.
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:46
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:60
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:233
Header file for FFmpegUtilities.
This class is a memory-based cache manager for Frame objects.
Definition: CacheMemory.h:50