OpenShot Library | libopenshot  0.2.3
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegReader.h"
32 
33 using namespace openshot;
34 
36  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
41  packet(NULL) {
42 
43  // Initialize FFMpeg, and register all formats and codecs
46 
47  // Init cache
51 
52  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
53  Open();
54  Close();
55 }
56 
57 FFmpegReader::FFmpegReader(string path, bool inspect_reader)
58  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
59  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
60  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
61  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
62  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
63  packet(NULL) {
64 
65  // Initialize FFMpeg, and register all formats and codecs
68 
69  // Init cache
73 
74  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
75  if (inspect_reader) {
76  Open();
77  Close();
78  }
79 }
80 
82  if (is_open)
83  // Auto close reader if not already done
84  Close();
85 }
86 
87 // This struct holds the associated video frame and starting sample # for an audio packet.
88 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount)
89 {
90  // Is frame even close to this one?
91  if (abs(location.frame - frame) >= 2)
92  // This is too far away to be considered
93  return false;
94 
95  // Note that samples_per_frame can vary slightly frame to frame when the
96  // audio sampling rate is not an integer multiple of the video fps.
97  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
98  if (abs(diff) <= amount)
99  // close
100  return true;
101 
102  // not close
103  return false;
104 }
105 
107 {
108  // Open reader if not already open
109  if (!is_open)
110  {
111  // Initialize format context
112  pFormatCtx = NULL;
113 
114  // Open video file
115  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
116  throw InvalidFile("File could not be opened.", path);
117 
118  // Retrieve stream information
119  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
120  throw NoStreamsFound("No streams found in file.", path);
121 
122  videoStream = -1;
123  audioStream = -1;
124  // Loop through each stream, and identify the video and audio stream index
125  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
126  {
127  // Is this a video stream?
128  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
129  videoStream = i;
130  }
131  // Is this an audio stream?
132  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
133  audioStream = i;
134  }
135  }
136  if (videoStream == -1 && audioStream == -1)
137  throw NoStreamsFound("No video or audio streams found in this file.", path);
138 
139  // Is there a video stream?
140  if (videoStream != -1)
141  {
142  // Set the stream index
143  info.video_stream_index = videoStream;
144 
145  // Set the codec and codec context pointers
146  pStream = pFormatCtx->streams[videoStream];
147 
148  // Find the codec ID from stream
149  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
150 
151  // Get codec and codec context from stream
152  AVCodec *pCodec = avcodec_find_decoder(codecId);
153  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
154 
155  // Set number of threads equal to number of processors (not to exceed 16)
156  pCodecCtx->thread_count = min(FF_NUM_PROCESSORS, 16);
157 
158  if (pCodec == NULL) {
159  throw InvalidCodec("A valid video codec could not be found for this file.", path);
160  }
161 
162  // Init options
163  AVDictionary *opts = NULL;
164  av_dict_set(&opts, "strict", "experimental", 0);
165 
166  // Open video codec
167  if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
168  throw InvalidCodec("A video codec was found, but could not be opened.", path);
169 
170  // Free options
171  av_dict_free(&opts);
172 
173  // Update the File Info struct with video details (if a video stream is found)
174  UpdateVideoInfo();
175  }
176 
177  // Is there an audio stream?
178  if (audioStream != -1)
179  {
180  // Set the stream index
181  info.audio_stream_index = audioStream;
182 
183  // Get a pointer to the codec context for the audio stream
184  aStream = pFormatCtx->streams[audioStream];
185 
186  // Find the codec ID from stream
187  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
188 
189  // Get codec and codec context from stream
190  AVCodec *aCodec = avcodec_find_decoder(codecId);
191  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
192 
193  // Set number of threads equal to number of processors (not to exceed 16)
194  aCodecCtx->thread_count = min(FF_NUM_PROCESSORS, 16);
195 
196  if (aCodec == NULL) {
197  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
198  }
199 
200  // Init options
201  AVDictionary *opts = NULL;
202  av_dict_set(&opts, "strict", "experimental", 0);
203 
204  // Open audio codec
205  if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
206  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
207 
208  // Free options
209  av_dict_free(&opts);
210 
211  // Update the File Info struct with audio details (if an audio stream is found)
212  UpdateAudioInfo();
213  }
214 
215  // Add format metadata (if any)
216  AVDictionaryEntry *tag = NULL;
217  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
218  QString str_key = tag->key;
219  QString str_value = tag->value;
220  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
221  }
222 
223  // Init previous audio location to zero
224  previous_packet_location.frame = -1;
225  previous_packet_location.sample_start = 0;
226 
227  // Adjust cache size based on size of frame and audio
231 
232  // Mark as "open"
233  is_open = true;
234  }
235 }
236 
238 {
239  // Close all objects, if reader is 'open'
240  if (is_open)
241  {
242  // Mark as "closed"
243  is_open = false;
244 
245  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
246 
247  // Close the codec
248  if (info.has_video)
249  {
250  avcodec_flush_buffers(pCodecCtx);
251  AV_FREE_CONTEXT(pCodecCtx);
252  }
253  if (info.has_audio)
254  {
255  avcodec_flush_buffers(aCodecCtx);
256  AV_FREE_CONTEXT(aCodecCtx);
257  }
258 
259  // Clear final cache
260  final_cache.Clear();
261  working_cache.Clear();
262  missing_frames.Clear();
263 
264  // Clear processed lists
265  {
266  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
267  processed_video_frames.clear();
268  processed_audio_frames.clear();
269  processing_video_frames.clear();
270  processing_audio_frames.clear();
271  missing_audio_frames.clear();
272  missing_video_frames.clear();
273  missing_audio_frames_source.clear();
274  missing_video_frames_source.clear();
275  checked_frames.clear();
276  }
277 
278  // Close the video file
279  avformat_close_input(&pFormatCtx);
280  av_freep(&pFormatCtx);
281 
282  // Reset some variables
283  last_frame = 0;
284  largest_frame_processed = 0;
285  seek_audio_frame_found = 0;
286  seek_video_frame_found = 0;
287  current_video_frame = 0;
288  has_missing_frames = false;
289  }
290 }
291 
292 void FFmpegReader::UpdateAudioInfo()
293 {
294  // Set values of FileInfo struct
295  info.has_audio = true;
296  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
297  info.acodec = aCodecCtx->codec->name;
298  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
299  if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
300  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout( AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels );
301  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
302  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
303  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
304 
305  // Set audio timebase
306  info.audio_timebase.num = aStream->time_base.num;
307  info.audio_timebase.den = aStream->time_base.den;
308 
309  // Get timebase of audio stream (if valid) and greater than the current duration
310  if (aStream->duration > 0.0f && aStream->duration > info.duration)
311  info.duration = aStream->duration * info.audio_timebase.ToDouble();
312 
313  // Check for an invalid video length
314  if (info.has_video && info.video_length <= 0)
315  {
316  // Calculate the video length from the audio duration
318  }
319 
320  // Set video timebase (if no video stream was found)
321  if (!info.has_video)
322  {
323  // Set a few important default video settings (so audio can be divided into frames)
324  info.fps.num = 24;
325  info.fps.den = 1;
326  info.video_timebase.num = 1;
327  info.video_timebase.den = 24;
329  info.width = 720;
330  info.height = 480;
331  }
332 
333  // Fix invalid video lengths for certain types of files (MP3 for example)
334  if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) {
336  }
337 
338  // Add audio metadata (if any found)
339  AVDictionaryEntry *tag = NULL;
340  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
341  QString str_key = tag->key;
342  QString str_value = tag->value;
343  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
344  }
345 }
346 
347 void FFmpegReader::UpdateVideoInfo()
348 {
349  if (check_fps)
350  // Already initialized all the video metadata, no reason to do it again
351  return;
352 
353  // Set values of FileInfo struct
354  info.has_video = true;
355  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
356  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
357  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
358  info.vcodec = pCodecCtx->codec->name;
359  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
360 
361  // set frames per second (fps)
362  info.fps.num = pStream->avg_frame_rate.num;
363  info.fps.den = pStream->avg_frame_rate.den;
364 
365  if (pStream->sample_aspect_ratio.num != 0)
366  {
367  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
368  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
369  }
370  else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0)
371  {
372  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
373  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
374  }
375  else
376  {
377  info.pixel_ratio.num = 1;
378  info.pixel_ratio.den = 1;
379  }
380  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
381 
382  // Calculate the DAR (display aspect ratio)
384 
385  // Reduce size fraction
386  size.Reduce();
387 
388  // Set the ratio based on the reduced fraction
389  info.display_ratio.num = size.num;
390  info.display_ratio.den = size.den;
391 
392  // Set the video timebase
393  info.video_timebase.num = pStream->time_base.num;
394  info.video_timebase.den = pStream->time_base.den;
395 
396  // Set the duration in seconds, and video length (# of frames)
397  info.duration = pStream->duration * info.video_timebase.ToDouble();
398 
399  // Check for valid duration (if found)
400  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
401  // Use the format's duration
402  info.duration = pFormatCtx->duration / AV_TIME_BASE;
403 
404  // Calculate duration from filesize and bitrate (if any)
405  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
406  // Estimate from bitrate, total bytes, and framerate
408 
409  // No duration found in stream of file
410  if (info.duration <= 0.0f)
411  {
412  // No duration is found in the video stream
413  info.duration = -1;
414  info.video_length = -1;
415  is_duration_known = false;
416  }
417  else
418  {
419  // Yes, a duration was found
420  is_duration_known = true;
421 
422  // Calculate number of frames
424  }
425 
426  // Override an invalid framerate
427  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
428  // Calculate FPS, duration, video bit rate, and video length manually
429  // by scanning through all the video stream packets
430  CheckFPS();
431  }
432 
433  // Add video metadata (if any)
434  AVDictionaryEntry *tag = NULL;
435  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
436  QString str_key = tag->key;
437  QString str_value = tag->value;
438  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
439  }
440 }
441 
442 
443 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame)
444 {
445  // Check for open reader (or throw exception)
446  if (!is_open)
447  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
448 
449  // Adjust for a requested frame that is too small or too large
450  if (requested_frame < 1)
451  requested_frame = 1;
452  if (requested_frame > info.video_length && is_duration_known)
453  requested_frame = info.video_length;
454  if (info.has_video && info.video_length == 0)
455  // Invalid duration of video file
456  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
457 
458  // Debug output
459  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame, "", -1, "", -1, "", -1, "", -1);
460 
461  // Check the cache for this frame
462  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
463  if (frame) {
464  // Debug output
465  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
466 
467  // Return the cached frame
468  return frame;
469  }
470  else
471  {
472  #pragma omp critical (ReadStream)
473  {
474  // Check the cache a 2nd time (due to a potential previous lock)
475  frame = final_cache.GetFrame(requested_frame);
476  if (frame) {
477  // Debug output
478  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
479 
480  // Return the cached frame
481  }
482  else {
483  // Frame is not in cache
484  // Reset seek count
485  seek_count = 0;
486 
487  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
488  if (last_frame == 0 && requested_frame != 1)
489  // Get first frame
490  ReadStream(1);
491 
492  // Are we within X frames of the requested frame?
493  int64_t diff = requested_frame - last_frame;
494  if (diff >= 1 && diff <= 20)
495  {
496  // Continue walking the stream
497  frame = ReadStream(requested_frame);
498  }
499  else
500  {
501  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
502  if (enable_seek)
503  // Only seek if enabled
504  Seek(requested_frame);
505 
506  else if (!enable_seek && diff < 0)
507  {
508  // Start over, since we can't seek, and the requested frame is smaller than our position
509  Close();
510  Open();
511  }
512 
513  // Then continue walking the stream
514  frame = ReadStream(requested_frame);
515  }
516  }
517  } //omp critical
518  return frame;
519  }
520 }
521 
522 // Read the stream until we find the requested Frame
523 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
524 {
525  // Allocate video frame
526  bool end_of_stream = false;
527  bool check_seek = false;
528  bool frame_finished = false;
529  int packet_error = -1;
530 
531  // Minimum number of packets to process (for performance reasons)
532  int packets_processed = 0;
533  int minimum_packets = OPEN_MP_NUM_PROCESSORS;
534  int max_packets = 4096;
535 
536  // Set the number of threads in OpenMP
537  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
538  // Allow nested OpenMP sections
539  omp_set_nested(true);
540 
541  // Debug output
542  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "OPEN_MP_NUM_PROCESSORS", OPEN_MP_NUM_PROCESSORS, "", -1, "", -1, "", -1, "", -1);
543 
544  #pragma omp parallel
545  {
546  #pragma omp single
547  {
548  // Loop through the stream until the correct frame is found
549  while (true)
550  {
551  // Get the next packet into a local variable called packet
552  packet_error = GetNextPacket();
553 
554  int processing_video_frames_size = 0;
555  int processing_audio_frames_size = 0;
556  {
557  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
558  processing_video_frames_size = processing_video_frames.size();
559  processing_audio_frames_size = processing_audio_frames.size();
560  }
561 
562  // Wait if too many frames are being processed
563  while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
564  usleep(2500);
565  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
566  processing_video_frames_size = processing_video_frames.size();
567  processing_audio_frames_size = processing_audio_frames.size();
568  }
569 
570  // Get the next packet (if any)
571  if (packet_error < 0)
572  {
573  // Break loop when no more packets found
574  end_of_stream = true;
575  break;
576  }
577 
578  // Debug output
579  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking);
580 
581  // Video packet
582  if (info.has_video && packet->stream_index == videoStream)
583  {
584  // Reset this counter, since we have a video packet
585  num_packets_since_video_frame = 0;
586 
587  // Check the status of a seek (if any)
588  if (is_seeking)
589  #pragma omp critical (openshot_seek)
590  check_seek = CheckSeek(true);
591  else
592  check_seek = false;
593 
594  if (check_seek) {
595  // Jump to the next iteration of this loop
596  continue;
597  }
598 
599  // Get the AVFrame from the current packet
600  frame_finished = GetAVFrame();
601 
602  // Check if the AVFrame is finished and set it
603  if (frame_finished)
604  {
605  // Update PTS / Frame Offset (if any)
606  UpdatePTSOffset(true);
607 
608  // Process Video Packet
609  ProcessVideoPacket(requested_frame);
610 
611  if (openshot::Settings::Instance()->WAIT_FOR_VIDEO_PROCESSING_TASK) {
612  // Wait on each OMP task to complete before moving on to the next one. This slows
613  // down processing considerably, but might be more stable on some systems.
614  #pragma omp taskwait
615  }
616  }
617 
618  }
619  // Audio packet
620  else if (info.has_audio && packet->stream_index == audioStream)
621  {
622  // Increment this (to track # of packets since the last video packet)
623  num_packets_since_video_frame++;
624 
625  // Check the status of a seek (if any)
626  if (is_seeking)
627  #pragma omp critical (openshot_seek)
628  check_seek = CheckSeek(false);
629  else
630  check_seek = false;
631 
632  if (check_seek) {
633  // Jump to the next iteration of this loop
634  continue;
635  }
636 
637  // Update PTS / Frame Offset (if any)
638  UpdatePTSOffset(false);
639 
640  // Determine related video frame and starting sample # from audio PTS
641  AudioLocation location = GetAudioPTSLocation(packet->pts);
642 
643  // Process Audio Packet
644  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
645  }
646 
647  // Check if working frames are 'finished'
648  if (!is_seeking) {
649  // Check for final frames
650  CheckWorkingFrames(false, requested_frame);
651  }
652 
653  // Check if requested 'final' frame is available
654  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
655 
656  // Increment frames processed
657  packets_processed++;
658 
659  // Break once the frame is found
660  if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
661  break;
662 
663  } // end while
664 
665  } // end omp single
666 
667  } // end omp parallel
668 
669  // Debug output
670  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count(), "", -1, "", -1);
671 
672  // End of stream?
673  if (end_of_stream)
674  // Mark the any other working frames as 'finished'
675  CheckWorkingFrames(end_of_stream, requested_frame);
676 
677  // Return requested frame (if found)
678  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
679  if (frame)
680  // Return prepared frame
681  return frame;
682  else {
683 
684  // Check if largest frame is still cached
685  frame = final_cache.GetFrame(largest_frame_processed);
686  if (frame) {
687  // return the largest processed frame (assuming it was the last in the video file)
688  return frame;
689  }
690  else {
691  // The largest processed frame is no longer in cache, return a blank frame
692  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
693  f->AddColor(info.width, info.height, "#000");
694  return f;
695  }
696  }
697 
698 }
699 
700 // Get the next packet (if any)
701 int FFmpegReader::GetNextPacket()
702 {
703  int found_packet = 0;
704  AVPacket *next_packet = new AVPacket();
705  found_packet = av_read_frame(pFormatCtx, next_packet);
706 
707  if (packet) {
708  // Remove previous packet before getting next one
709  RemoveAVPacket(packet);
710  packet = NULL;
711  }
712 
713  if (found_packet >= 0)
714  {
715  // Update current packet pointer
716  packet = next_packet;
717  }
718 
719  // Return if packet was found (or error number)
720  return found_packet;
721 }
722 
723 // Get an AVFrame (if any)
724 bool FFmpegReader::GetAVFrame()
725 {
726  int frameFinished = -1;
727  int ret = 0;
728 
729  // Decode video frame
730  AVFrame *next_frame = AV_ALLOCATE_FRAME();
731  #pragma omp critical (packet_cache)
732  {
733  #if IS_FFMPEG_3_2
734  frameFinished = 0;
735  ret = avcodec_send_packet(pCodecCtx, packet);
736  if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
737  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
738  }
739  else {
740  pFrame = new AVFrame();
741  while (ret >= 0) {
742  ret = avcodec_receive_frame(pCodecCtx, next_frame);
743  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
744  break;
745  }
746  // TODO also handle possible further frames
747  // Use only the first frame like avcodec_decode_video2
748  if (frameFinished == 0 ) {
749  frameFinished = 1;
750  av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
751  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
752  (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
753  if (!check_interlace) {
754  check_interlace = true;
755  info.interlaced_frame = next_frame->interlaced_frame;
756  info.top_field_first = next_frame->top_field_first;
757  }
758  }
759  }
760  }
761  #else
762  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
763 
764  // is frame finished
765  if (frameFinished) {
766  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
767  // must make a copy of the image data before this method is called again.
768  pFrame = AV_ALLOCATE_FRAME();
769  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
770  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
771  info.height);
772 
773  // Detect interlaced frame (only once)
774  if (!check_interlace) {
775  check_interlace = true;
776  info.interlaced_frame = next_frame->interlaced_frame;
777  info.top_field_first = next_frame->top_field_first;
778  }
779  }
780  #endif
781  }
782 
783  // deallocate the frame
784  AV_FREE_FRAME(&next_frame);
785 
786  // Did we get a video frame?
787  return frameFinished;
788 }
789 
790 // Check the current seek position and determine if we need to seek again
791 bool FFmpegReader::CheckSeek(bool is_video)
792 {
793  // Are we seeking for a specific frame?
794  if (is_seeking)
795  {
796  // Determine if both an audio and video packet have been decoded since the seek happened.
797  // If not, allow the ReadStream method to keep looping
798  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
799  return false;
800 
801  // Check for both streams
802  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
803  return false;
804 
805  // Determine max seeked frame
806  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
807  if (seek_video_frame_found > max_seeked_frame)
808  max_seeked_frame = seek_video_frame_found;
809 
810  // determine if we are "before" the requested frame
811  if (max_seeked_frame >= seeking_frame)
812  {
813  // SEEKED TOO FAR
814  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
815 
816  // Seek again... to the nearest Keyframe
817  Seek(seeking_frame - (10 * seek_count * seek_count));
818  }
819  else
820  {
821  // SEEK WORKED
822  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
823 
824  // Seek worked, and we are "before" the requested frame
825  is_seeking = false;
826  seeking_frame = 0;
827  seeking_pts = -1;
828  }
829  }
830 
831  // return the pts to seek to (if any)
832  return is_seeking;
833 }
834 
835 // Process a video packet
836 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame)
837 {
838  // Calculate current frame #
839  int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
840 
841  // Track 1st video packet after a successful seek
842  if (!seek_video_frame_found && is_seeking)
843  seek_video_frame_found = current_frame;
844 
845  // Are we close enough to decode the frame? and is this frame # valid?
846  if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
847  {
848  // Remove frame and packet
849  RemoveAVFrame(pFrame);
850 
851  // Debug output
852  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
853 
854  // Skip to next frame without decoding or caching
855  return;
856  }
857 
858  // Debug output
859  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
860 
861  // Init some things local (for OpenMP)
862  PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
863  int height = info.height;
864  int width = info.width;
865  int64_t video_length = info.video_length;
866  AVFrame *my_frame = pFrame;
867 
868  // Add video frame to list of processing video frames
869  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
870  processing_video_frames[current_frame] = current_frame;
871 
872  #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt)
873  {
874  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
875  AVFrame *pFrameRGB = NULL;
876  int numBytes;
877  uint8_t *buffer = NULL;
878 
879  // Allocate an AVFrame structure
880  pFrameRGB = AV_ALLOCATE_FRAME();
881  if (pFrameRGB == NULL)
882  throw OutOfBoundsFrame("Convert Image Broke!", current_frame, video_length);
883 
884  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
885  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
886  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
887  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
888  // the future.
889  int max_width = Settings::Instance()->MAX_WIDTH;
890  if (max_width <= 0)
891  max_width = info.width;
892  int max_height = Settings::Instance()->MAX_HEIGHT;
893  if (max_height <= 0)
894  max_height = info.height;
895 
896  Clip* parent = (Clip*) GetClip();
897  if (parent) {
898  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
899  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
900  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
901  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
902  max_width = max(float(max_width), max_width * max_scale_x);
903  max_height = max(float(max_height), max_height * max_scale_y);
904 
905  } else if (parent->scale == SCALE_CROP) {
906  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
907  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
908  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
909  QSize width_size(max_width * max_scale_x,
910  round(max_width / (float(info.width) / float(info.height))));
911  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
912  max_height * max_scale_y);
913  // respect aspect ratio
914  if (width_size.width() >= max_width && width_size.height() >= max_height) {
915  max_width = max(max_width, width_size.width());
916  max_height = max(max_height, width_size.height());
917  }
918  else {
919  max_width = max(max_width, height_size.width());
920  max_height = max(max_height, height_size.height());
921  }
922 
923  } else {
924  // No scaling, use original image size (slower)
925  max_width = info.width;
926  max_height = info.height;
927  }
928  }
929 
930  // Determine if image needs to be scaled (for performance reasons)
931  int original_height = height;
932  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
933  // Override width and height (but maintain aspect ratio)
934  float ratio = float(width) / float(height);
935  int possible_width = round(max_height * ratio);
936  int possible_height = round(max_width / ratio);
937 
938  if (possible_width <= max_width) {
939  // use calculated width, and max_height
940  width = possible_width;
941  height = max_height;
942  } else {
943  // use max_width, and calculated height
944  width = max_width;
945  height = possible_height;
946  }
947  }
948 
949  // Determine required buffer size and allocate buffer
950  numBytes = AV_GET_IMAGE_SIZE(PIX_FMT_RGBA, width, height);
951 
952  #pragma omp critical (video_buffer)
953  buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
954 
955  // Copy picture data from one AVFrame (or AVPicture) to another one.
956  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
957 
958  int scale_mode = SWS_FAST_BILINEAR;
959  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
960  scale_mode = SWS_LANCZOS;
961  }
962  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
963  height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
964 
965  // Resize / Convert to RGB
966  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
967  original_height, pFrameRGB->data, pFrameRGB->linesize);
968 
969  // Create or get the existing frame object
970  std::shared_ptr<Frame> f = CreateFrame(current_frame);
971 
972  // Add Image data to frame
973  f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
974 
975  // Update working cache
976  working_cache.Add(f);
977 
978  // Keep track of last last_video_frame
979  #pragma omp critical (video_buffer)
980  last_video_frame = f;
981 
982  // Free the RGB image
983  av_free(buffer);
984  AV_FREE_FRAME(&pFrameRGB);
985 
986  // Remove frame and packet
987  RemoveAVFrame(my_frame);
988  sws_freeContext(img_convert_ctx);
989 
990  // Remove video frame from list of processing video frames
991  {
992  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
993  processing_video_frames.erase(current_frame);
994  processed_video_frames[current_frame] = current_frame;
995  }
996 
997  // Debug output
998  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "", -1, "", -1, "", -1);
999 
1000  } // end omp task
1001 
1002 }
1003 
1004 // Process an audio packet
1005 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample)
1006 {
1007  // Track 1st audio packet after a successful seek
1008  if (!seek_audio_frame_found && is_seeking)
1009  seek_audio_frame_found = target_frame;
1010 
1011  // Are we close enough to decode the frame's audio?
1012  if (target_frame < (requested_frame - 20))
1013  {
1014  // Debug output
1015  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
1016 
1017  // Skip to next frame without decoding or caching
1018  return;
1019  }
1020 
1021  // Debug output
1022  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
1023 
1024  // Init an AVFrame to hold the decoded audio samples
1025  int frame_finished = 0;
1026  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
1027  AV_RESET_FRAME(audio_frame);
1028 
1029  int packet_samples = 0;
1030  int data_size = 0;
1031 
1032  // re-initialize buffer size (it gets changed in the avcodec_decode_audio2 method call)
1034  #pragma omp critical (ProcessAudioPacket)
1035  {
1036  #if IS_FFMPEG_3_2
1037  int ret = 0;
1038  frame_finished = 1;
1039  while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
1040  frame_finished = 0;
1041  ret = avcodec_send_packet(aCodecCtx, packet);
1042  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1043  avcodec_send_packet(aCodecCtx, NULL);
1044  break;
1045  }
1046  if (ret >= 0)
1047  packet->size = 0;
1048  ret = avcodec_receive_frame(aCodecCtx, audio_frame);
1049  if (ret >= 0)
1050  frame_finished = 1;
1051  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1052  avcodec_flush_buffers(aCodecCtx);
1053  ret = 0;
1054  }
1055  if (ret >= 0) {
1056  ret = frame_finished;
1057  }
1058  }
1059  if (!packet->data && !frame_finished)
1060  {
1061  ret = -1;
1062  }
1063  #else
1064  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1065 #endif
1066  }
1067 
1068  if (frame_finished) {
1069 
1070  // determine how many samples were decoded
1071  int planar = av_sample_fmt_is_planar((AVSampleFormat)AV_GET_CODEC_PIXEL_FORMAT(aStream, aCodecCtx));
1072  int plane_size = -1;
1073  data_size = av_samples_get_buffer_size(&plane_size,
1074  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels,
1075  audio_frame->nb_samples,
1076  (AVSampleFormat)(AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1077 
1078  // Calculate total number of samples
1079  packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1080  }
1081 
1082  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1083  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1084 
1085  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
1086  int64_t adjusted_pts = packet->pts + audio_pts_offset;
1087  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
1088  double sample_seconds = double(pts_total) / info.sample_rate;
1089 
1090  // Debug output
1091  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
1092  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
1093 
1094  // DEBUG (FOR AUDIO ISSUES)
1095  prev_pts = adjusted_pts;
1096  pts_total += pts_remaining_samples;
1097  pts_counter++;
1098  prev_samples = pts_remaining_samples;
1099 
1100  // Add audio frame to list of processing audio frames
1101  {
1102  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1103  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1104  }
1105 
1106  while (pts_remaining_samples)
1107  {
1108  // Get Samples per frame (for this frame number)
1109  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1110 
1111  // Calculate # of samples to add to this frame
1112  int samples = samples_per_frame - previous_packet_location.sample_start;
1113  if (samples > pts_remaining_samples)
1114  samples = pts_remaining_samples;
1115 
1116  // Decrement remaining samples
1117  pts_remaining_samples -= samples;
1118 
1119  if (pts_remaining_samples > 0) {
1120  // next frame
1121  previous_packet_location.frame++;
1122  previous_packet_location.sample_start = 0;
1123 
1124  // Add audio frame to list of processing audio frames
1125  {
1126  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1127  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1128  }
1129 
1130  } else {
1131  // Increment sample start
1132  previous_packet_location.sample_start += samples;
1133  }
1134  }
1135 
1136 
1137  // Allocate audio buffer
1138  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE];
1139 
1140  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16, "", -1);
1141 
1142  // Create output frame
1143  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1144  AV_RESET_FRAME(audio_converted);
1145  audio_converted->nb_samples = audio_frame->nb_samples;
1146  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1147 
1148  SWRCONTEXT *avr = NULL;
1149  int nb_samples = 0;
1150 
1151  // setup resample context
1152  avr = SWR_ALLOC();
1153  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1154  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1155  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1156  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1157  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1158  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1159  av_opt_set_int(avr, "in_channels", info.channels, 0);
1160  av_opt_set_int(avr, "out_channels", info.channels, 0);
1161  int r = SWR_INIT(avr);
1162 
1163  // Convert audio samples
1164  nb_samples = SWR_CONVERT(avr, // audio resample context
1165  audio_converted->data, // output data pointers
1166  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1167  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1168  audio_frame->data, // input data pointers
1169  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1170  audio_frame->nb_samples); // number of input samples to convert
1171 
1172  // Copy audio samples over original samples
1173  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1174 
1175  // Deallocate resample buffer
1176  SWR_CLOSE(avr);
1177  SWR_FREE(&avr);
1178  avr = NULL;
1179 
1180  // Free AVFrames
1181  av_free(audio_converted->data[0]);
1182  AV_FREE_FRAME(&audio_converted);
1183 
1184  int64_t starting_frame_number = -1;
1185  bool partial_frame = true;
1186  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
1187  {
1188  // Array of floats (to hold samples for each channel)
1189  starting_frame_number = target_frame;
1190  int channel_buffer_size = packet_samples / info.channels;
1191  float *channel_buffer = new float[channel_buffer_size];
1192 
1193  // Init buffer array
1194  for (int z = 0; z < channel_buffer_size; z++)
1195  channel_buffer[z] = 0.0f;
1196 
1197  // Loop through all samples and add them to our Frame based on channel.
1198  // Toggle through each channel number, since channel data is stored like (left right left right)
1199  int channel = 0;
1200  int position = 0;
1201  for (int sample = 0; sample < packet_samples; sample++)
1202  {
1203  // Only add samples for current channel
1204  if (channel_filter == channel)
1205  {
1206  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1207  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1208 
1209  // Increment audio position
1210  position++;
1211  }
1212 
1213  // increment channel (if needed)
1214  if ((channel + 1) < info.channels)
1215  // move to next channel
1216  channel ++;
1217  else
1218  // reset channel
1219  channel = 0;
1220  }
1221 
1222  // Loop through samples, and add them to the correct frames
1223  int start = starting_sample;
1224  int remaining_samples = channel_buffer_size;
1225  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1226  while (remaining_samples > 0)
1227  {
1228  // Get Samples per frame (for this frame number)
1229  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1230 
1231  // Calculate # of samples to add to this frame
1232  int samples = samples_per_frame - start;
1233  if (samples > remaining_samples)
1234  samples = remaining_samples;
1235 
1236  // Create or get the existing frame object
1237  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1238 
1239  // Determine if this frame was "partially" filled in
1240  if (samples_per_frame == start + samples)
1241  partial_frame = false;
1242  else
1243  partial_frame = true;
1244 
1245  // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent
1246  // some louder samples from maxing out at 1.0 (not sure why this happens)
1247  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1248 
1249  // Debug output
1250  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1251 
1252  // Add or update cache
1253  working_cache.Add(f);
1254 
1255  // Decrement remaining samples
1256  remaining_samples -= samples;
1257 
1258  // Increment buffer (to next set of samples)
1259  if (remaining_samples > 0)
1260  iterate_channel_buffer += samples;
1261 
1262  // Increment frame number
1263  starting_frame_number++;
1264 
1265  // Reset starting sample #
1266  start = 0;
1267  }
1268 
1269  // clear channel buffer
1270  delete[] channel_buffer;
1271  channel_buffer = NULL;
1272  iterate_channel_buffer = NULL;
1273  }
1274 
1275  // Clean up some arrays
1276  delete[] audio_buf;
1277  audio_buf = NULL;
1278 
1279  // Remove audio frame from list of processing audio frames
1280  {
1281  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1282  // Update all frames as completed
1283  for (int64_t f = target_frame; f < starting_frame_number; f++) {
1284  // Remove the frame # from the processing list. NOTE: If more than one thread is
1285  // processing this frame, the frame # will be in this list multiple times. We are only
1286  // removing a single instance of it here.
1287  processing_audio_frames.erase(processing_audio_frames.find(f));
1288 
1289  // Check and see if this frame is also being processed by another thread
1290  if (processing_audio_frames.count(f) == 0)
1291  // No other thread is processing it. Mark the audio as processed (final)
1292  processed_audio_frames[f] = f;
1293  }
1294 
1295  if (target_frame == starting_frame_number) {
1296  // This typically never happens, but just in case, remove the currently processing number
1297  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1298  }
1299  }
1300 
1301  // Free audio frame
1302  AV_FREE_FRAME(&audio_frame);
1303 
1304  // Debug output
1305  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1, "", -1, "", -1, "", -1);
1306 
1307 }
1308 
1309 
1310 
1311 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1312 void FFmpegReader::Seek(int64_t requested_frame)
1313 {
1314  // Adjust for a requested frame that is too small or too large
1315  if (requested_frame < 1)
1316  requested_frame = 1;
1317  if (requested_frame > info.video_length)
1318  requested_frame = info.video_length;
1319 
1320  int processing_video_frames_size = 0;
1321  int processing_audio_frames_size = 0;
1322  {
1323  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1324  processing_video_frames_size = processing_video_frames.size();
1325  processing_audio_frames_size = processing_audio_frames.size();
1326  }
1327 
1328  // Debug output
1329  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "video_pts_offset", video_pts_offset);
1330 
1331  // Wait for any processing frames to complete
1332  while (processing_video_frames_size + processing_audio_frames_size > 0) {
1333  usleep(2500);
1334  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1335  processing_video_frames_size = processing_video_frames.size();
1336  processing_audio_frames_size = processing_audio_frames.size();
1337  }
1338 
1339  // Clear working cache (since we are seeking to another location in the file)
1340  working_cache.Clear();
1341  missing_frames.Clear();
1342 
1343  // Clear processed lists
1344  {
1345  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1346  processing_audio_frames.clear();
1347  processing_video_frames.clear();
1348  processed_video_frames.clear();
1349  processed_audio_frames.clear();
1350  missing_audio_frames.clear();
1351  missing_video_frames.clear();
1352  missing_audio_frames_source.clear();
1353  missing_video_frames_source.clear();
1354  checked_frames.clear();
1355  }
1356 
1357  // Reset the last frame variable
1358  last_frame = 0;
1359  current_video_frame = 0;
1360  largest_frame_processed = 0;
1361  num_checks_since_final = 0;
1362  num_packets_since_video_frame = 0;
1363  has_missing_frames = false;
1364  bool has_audio_override = info.has_audio;
1365  bool has_video_override = info.has_video;
1366 
1367  // Increment seek count
1368  seek_count++;
1369 
1370  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1371  int buffer_amount = max(OPEN_MP_NUM_PROCESSORS, 8);
1372  if (requested_frame - buffer_amount < 20)
1373  {
1374  // Close and re-open file (basically seeking to frame 1)
1375  Close();
1376  Open();
1377 
1378  // Update overrides (since closing and re-opening might update these)
1379  info.has_audio = has_audio_override;
1380  info.has_video = has_video_override;
1381 
1382  // Not actually seeking, so clear these flags
1383  is_seeking = false;
1384  if (seek_count == 1) {
1385  // Don't redefine this on multiple seek attempts for a specific frame
1386  seeking_frame = 1;
1387  seeking_pts = ConvertFrameToVideoPTS(1);
1388  }
1389  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1390  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1391  }
1392  else
1393  {
1394  // Seek to nearest key-frame (aka, i-frame)
1395  bool seek_worked = false;
1396  int64_t seek_target = 0;
1397 
1398  // Seek video stream (if any)
1399  if (!seek_worked && info.has_video)
1400  {
1401  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1402  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1403  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1404  } else
1405  {
1406  // VIDEO SEEK
1407  is_video_seek = true;
1408  seek_worked = true;
1409  }
1410  }
1411 
1412  // Seek audio stream (if not already seeked... and if an audio stream is found)
1413  if (!seek_worked && info.has_audio)
1414  {
1415  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1416  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1417  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1418  } else
1419  {
1420  // AUDIO SEEK
1421  is_video_seek = false;
1422  seek_worked = true;
1423  }
1424  }
1425 
1426  // Was the seek successful?
1427  if (seek_worked)
1428  {
1429  // Flush audio buffer
1430  if (info.has_audio)
1431  avcodec_flush_buffers(aCodecCtx);
1432 
1433  // Flush video buffer
1434  if (info.has_video)
1435  avcodec_flush_buffers(pCodecCtx);
1436 
1437  // Reset previous audio location to zero
1438  previous_packet_location.frame = -1;
1439  previous_packet_location.sample_start = 0;
1440 
1441  // init seek flags
1442  is_seeking = true;
1443  if (seek_count == 1) {
1444  // Don't redefine this on multiple seek attempts for a specific frame
1445  seeking_pts = seek_target;
1446  seeking_frame = requested_frame;
1447  }
1448  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1449  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1450 
1451  }
1452  else
1453  {
1454  // seek failed
1455  is_seeking = false;
1456  seeking_pts = 0;
1457  seeking_frame = 0;
1458 
1459  // dislable seeking for this reader (since it failed)
1460  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1461  enable_seek = false;
1462 
1463  // Close and re-open file (basically seeking to frame 1)
1464  Close();
1465  Open();
1466 
1467  // Update overrides (since closing and re-opening might update these)
1468  info.has_audio = has_audio_override;
1469  info.has_video = has_video_override;
1470  }
1471  }
1472 }
1473 
1474 // Get the PTS for the current video packet
1475 int64_t FFmpegReader::GetVideoPTS()
1476 {
1477  int64_t current_pts = 0;
1478  if(packet->dts != AV_NOPTS_VALUE)
1479  current_pts = packet->dts;
1480 
1481  // Return adjusted PTS
1482  return current_pts;
1483 }
1484 
1485 // Update PTS Offset (if any)
1486 void FFmpegReader::UpdatePTSOffset(bool is_video)
1487 {
1488  // Determine the offset between the PTS and Frame number (only for 1st frame)
1489  if (is_video)
1490  {
1491  // VIDEO PACKET
1492  if (video_pts_offset == 99999) // Has the offset been set yet?
1493  {
1494  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1495  video_pts_offset = 0 - max(GetVideoPTS(), (int64_t) info.video_timebase.ToInt() * 10);
1496 
1497  // debug output
1498  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1);
1499  }
1500  }
1501  else
1502  {
1503  // AUDIO PACKET
1504  if (audio_pts_offset == 99999) // Has the offset been set yet?
1505  {
1506  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1507  audio_pts_offset = 0 - max(packet->pts, (int64_t) info.audio_timebase.ToInt() * 10);
1508 
1509  // debug output
1510  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Audio)", "audio_pts_offset", audio_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1);
1511  }
1512  }
1513 }
1514 
1515 // Convert PTS into Frame Number
1516 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts)
1517 {
1518  // Apply PTS offset
1519  pts = pts + video_pts_offset;
1520  int64_t previous_video_frame = current_video_frame;
1521 
1522  // Get the video packet start time (in seconds)
1523  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1524 
1525  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1526  int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
1527 
1528  // Keep track of the expected video frame #
1529  if (current_video_frame == 0)
1530  current_video_frame = frame;
1531  else {
1532 
1533  // Sometimes frames are duplicated due to identical (or similar) timestamps
1534  if (frame == previous_video_frame) {
1535  // return -1 frame number
1536  frame = -1;
1537  }
1538  else
1539  // Increment expected frame
1540  current_video_frame++;
1541 
1542  if (current_video_frame < frame)
1543  // has missing frames
1544  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame, "", -1, "", -1, "", -1);
1545 
1546  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1547  // if we are missing a video frame.
1548  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1549  while (current_video_frame < frame) {
1550  if (!missing_video_frames.count(current_video_frame)) {
1551  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame, "", -1, "", -1, "", -1, "", -1);
1552  missing_video_frames.insert(pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1553  missing_video_frames_source.insert(pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1554  }
1555 
1556  // Mark this reader as containing missing frames
1557  has_missing_frames = true;
1558 
1559  // Increment current frame
1560  current_video_frame++;
1561  }
1562  }
1563 
1564  // Return frame #
1565  return frame;
1566 }
1567 
1568 // Convert Frame Number into Video PTS
1569 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number)
1570 {
1571  // Get timestamp of this frame (in seconds)
1572  double seconds = double(frame_number) / info.fps.ToDouble();
1573 
1574  // Calculate the # of video packets in this timestamp
1575  int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
1576 
1577  // Apply PTS offset (opposite)
1578  return video_pts - video_pts_offset;
1579 }
1580 
1581 // Convert Frame Number into Video PTS
1582 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number)
1583 {
1584  // Get timestamp of this frame (in seconds)
1585  double seconds = double(frame_number) / info.fps.ToDouble();
1586 
1587  // Calculate the # of audio packets in this timestamp
1588  int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
1589 
1590  // Apply PTS offset (opposite)
1591  return audio_pts - audio_pts_offset;
1592 }
1593 
1594 // Calculate Starting video frame and sample # for an audio PTS
1595 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
1596 {
1597  // Apply PTS offset
1598  pts = pts + audio_pts_offset;
1599 
1600  // Get the audio packet start time (in seconds)
1601  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1602 
1603  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1604  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1605 
1606  // Frame # as a whole number (no more decimals)
1607  int64_t whole_frame = int64_t(frame);
1608 
1609  // Remove the whole number, and only get the decimal of the frame
1610  double sample_start_percentage = frame - double(whole_frame);
1611 
1612  // Get Samples per frame
1613  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1614 
1615  // Calculate the sample # to start on
1616  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1617 
1618  // Protect against broken (i.e. negative) timestamps
1619  if (whole_frame < 1)
1620  whole_frame = 1;
1621  if (sample_start < 0)
1622  sample_start = 0;
1623 
1624  // Prepare final audio packet location
1625  AudioLocation location = {whole_frame, sample_start};
1626 
1627  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1628  if (previous_packet_location.frame != -1) {
1629  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1630  {
1631  int64_t orig_frame = location.frame;
1632  int orig_start = location.sample_start;
1633 
1634  // Update sample start, to prevent gaps in audio
1635  location.sample_start = previous_packet_location.sample_start;
1636  location.frame = previous_packet_location.frame;
1637 
1638  // Debug output
1639  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1);
1640 
1641  } else {
1642  // Debug output
1643  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1, "", -1);
1644 
1645  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1646  for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
1647  if (!missing_audio_frames.count(audio_frame)) {
1648  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame, "", -1, "", -1, "", -1);
1649  missing_audio_frames.insert(pair<int64_t, int64_t>(audio_frame, previous_packet_location.frame - 1));
1650  }
1651  }
1652  }
1653  }
1654 
1655  // Set previous location
1656  previous_packet_location = location;
1657 
1658  // Return the associated video frame and starting sample #
1659  return location;
1660 }
1661 
1662 // Create a new Frame (or return an existing one) and add it to the working queue.
1663 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame)
1664 {
1665  // Check working cache
1666  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
1667  if (!output)
1668  {
1669  // Create a new frame on the working cache
1670  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
1671  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
1672  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
1673  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
1674 
1675  working_cache.Add(output);
1676 
1677  // Set the largest processed frame (if this is larger)
1678  if (requested_frame > largest_frame_processed)
1679  largest_frame_processed = requested_frame;
1680  }
1681 
1682  // Return new frame
1683  return output;
1684 }
1685 
1686 // Determine if frame is partial due to seek
1687 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
1688 
1689  // Sometimes a seek gets partial frames, and we need to remove them
1690  bool seek_trash = false;
1691  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1692  if (seek_video_frame_found > max_seeked_frame)
1693  max_seeked_frame = seek_video_frame_found;
1694  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1695  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1696  seek_trash = true;
1697 
1698  return seek_trash;
1699 }
1700 
1701 // Check if a frame is missing and attempt to replace it's frame image (and
1702 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
1703 {
1704  // Lock
1705  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1706 
1707  // Init # of times this frame has been checked so far
1708  int checked_count = 0;
1709 
1710  // Increment check count for this frame (or init to 1)
1711  if (checked_frames.count(requested_frame) == 0)
1712  checked_frames[requested_frame] = 1;
1713  else
1714  checked_frames[requested_frame]++;
1715  checked_count = checked_frames[requested_frame];
1716 
1717  // Debug output
1718  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_count, "", -1, "", -1);
1719 
1720  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
1721  map<int64_t, int64_t>::iterator itr;
1722  bool found_missing_frame = false;
1723 
1724  // Special MP3 Handling (ignore more than 1 video frame)
1725  if (info.has_audio and info.has_video) {
1726  AVCodecID aCodecId = AV_FIND_DECODER_CODEC_ID(aStream);
1727  AVCodecID vCodecId = AV_FIND_DECODER_CODEC_ID(pStream);
1728  // If MP3 with single video frame, handle this special case by copying the previously
1729  // decoded image to the new frame. Otherwise, it will spend a huge amount of
1730  // CPU time looking for missing images for all the audio-only frames.
1731  if (checked_count > 8 && !missing_video_frames.count(requested_frame) &&
1732  !processing_audio_frames.count(requested_frame) && processed_audio_frames.count(requested_frame) &&
1733  last_frame && last_video_frame->has_image_data && aCodecId == AV_CODEC_ID_MP3 && (vCodecId == AV_CODEC_ID_MJPEGB || vCodecId == AV_CODEC_ID_MJPEG)) {
1734  missing_video_frames.insert(pair<int64_t, int64_t>(requested_frame, last_video_frame->number));
1735  missing_video_frames_source.insert(pair<int64_t, int64_t>(last_video_frame->number, requested_frame));
1736  missing_frames.Add(last_video_frame);
1737  }
1738  }
1739 
1740  // Check if requested video frame is a missing
1741  if (missing_video_frames.count(requested_frame)) {
1742  int64_t missing_source_frame = missing_video_frames.find(requested_frame)->second;
1743 
1744  // Increment missing source frame check count (or init to 1)
1745  if (checked_frames.count(missing_source_frame) == 0)
1746  checked_frames[missing_source_frame] = 1;
1747  else
1748  checked_frames[missing_source_frame]++;
1749 
1750  // Get the previous frame of this missing frame (if it's available in missing cache)
1751  std::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
1752  if (parent_frame == NULL) {
1753  parent_frame = final_cache.GetFrame(missing_source_frame);
1754  if (parent_frame != NULL) {
1755  // Add missing final frame to missing cache
1756  missing_frames.Add(parent_frame);
1757  }
1758  }
1759 
1760  // Create blank missing frame
1761  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1762 
1763  // Debug output
1764  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1765 
1766  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
1767  if (parent_frame != NULL) {
1768  // Debug output
1769  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1770 
1771  // Add this frame to the processed map (since it's already done)
1772  std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
1773  if (parent_image) {
1774  missing_frame->AddImage(std::shared_ptr<QImage>(new QImage(*parent_image)));
1775  processed_video_frames[missing_frame->number] = missing_frame->number;
1776  }
1777  }
1778  }
1779 
1780  // Check if requested audio frame is a missing
1781  if (missing_audio_frames.count(requested_frame)) {
1782 
1783  // Create blank missing frame
1784  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1785 
1786  // Get Samples per frame (for this frame number)
1787  int samples_per_frame = Frame::GetSamplesPerFrame(missing_frame->number, info.fps, info.sample_rate, info.channels);
1788 
1789  // Debug output
1790  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Add Silence for Missing Audio Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "samples_per_frame", samples_per_frame, "", -1, "", -1, "", -1);
1791 
1792  // Add this frame to the processed map (since it's already done)
1793  missing_frame->AddAudioSilence(samples_per_frame);
1794  processed_audio_frames[missing_frame->number] = missing_frame->number;
1795  }
1796 
1797  return found_missing_frame;
1798 }
1799 
1800 // Check the working queue, and move finished frames to the finished queue
1801 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame)
1802 {
1803  // Loop through all working queue frames
1804  bool checked_count_tripped = false;
1805  int max_checked_count = 80;
1806 
1807  // Check if requested frame is 'missing'
1808  CheckMissingFrame(requested_frame);
1809 
1810  while (true)
1811  {
1812  // Get the front frame of working cache
1813  std::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
1814 
1815  // Was a frame found?
1816  if (!f)
1817  // No frames found
1818  break;
1819 
1820  // Remove frames which are too old
1821  if (f && f->number < (requested_frame - (OPEN_MP_NUM_PROCESSORS * 2))) {
1822  working_cache.Remove(f->number);
1823  }
1824 
1825  // Check if this frame is 'missing'
1826  CheckMissingFrame(f->number);
1827 
1828  // Init # of times this frame has been checked so far
1829  int checked_count = 0;
1830  int checked_frames_size = 0;
1831 
1832  bool is_video_ready = false;
1833  bool is_audio_ready = false;
1834  { // limit scope of next few lines
1835  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1836  is_video_ready = processed_video_frames.count(f->number);
1837  is_audio_ready = processed_audio_frames.count(f->number);
1838 
1839  // Get check count for this frame
1840  checked_frames_size = checked_frames.size();
1841  if (!checked_count_tripped || f->number >= requested_frame)
1842  checked_count = checked_frames[f->number];
1843  else
1844  // Force checked count over the limit
1845  checked_count = max_checked_count;
1846  }
1847 
1848  if (previous_packet_location.frame == f->number && !end_of_stream)
1849  is_audio_ready = false; // don't finalize the last processed audio frame
1850  bool is_seek_trash = IsPartialFrame(f->number);
1851 
1852  // Adjust for available streams
1853  if (!info.has_video) is_video_ready = true;
1854  if (!info.has_audio) is_audio_ready = true;
1855 
1856  // Make final any frames that get stuck (for whatever reason)
1857  if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
1858  // Debug output
1859  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
1860 
1861  // Trigger checked count tripped mode (clear out all frames before requested frame)
1862  checked_count_tripped = true;
1863 
1864  if (info.has_video && !is_video_ready && last_video_frame) {
1865  // Copy image from last frame
1866  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1867  is_video_ready = true;
1868  }
1869 
1870  if (info.has_audio && !is_audio_ready) {
1871  // Mark audio as processed, and indicate the frame has audio data
1872  is_audio_ready = true;
1873  }
1874  }
1875 
1876  // Debug output
1877  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
1878 
1879  // Check if working frame is final
1880  if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash)
1881  {
1882  // Debug output
1883  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "end_of_stream", end_of_stream);
1884 
1885  if (!is_seek_trash)
1886  {
1887  // Add missing image (if needed - sometimes end_of_stream causes frames with only audio)
1888  if (info.has_video && !is_video_ready && last_video_frame)
1889  // Copy image from last frame
1890  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1891 
1892  // Reset counter since last 'final' frame
1893  num_checks_since_final = 0;
1894 
1895  // Move frame to final cache
1896  final_cache.Add(f);
1897 
1898  // Add to missing cache (if another frame depends on it)
1899  {
1900  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1901  if (missing_video_frames_source.count(f->number)) {
1902  // Debug output
1903  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1);
1904  missing_frames.Add(f);
1905  }
1906 
1907  // Remove from 'checked' count
1908  checked_frames.erase(f->number);
1909  }
1910 
1911  // Remove frame from working cache
1912  working_cache.Remove(f->number);
1913 
1914  // Update last frame processed
1915  last_frame = f->number;
1916 
1917  } else {
1918  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
1919  working_cache.Remove(f->number);
1920  }
1921  }
1922  else
1923  // Stop looping
1924  break;
1925  }
1926 }
1927 
1928 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
1929 void FFmpegReader::CheckFPS()
1930 {
1931  check_fps = true;
1932 
1933  int first_second_counter = 0;
1934  int second_second_counter = 0;
1935  int third_second_counter = 0;
1936  int forth_second_counter = 0;
1937  int fifth_second_counter = 0;
1938  int frames_detected = 0;
1939  int64_t pts = 0;
1940 
1941  // Loop through the stream
1942  while (true)
1943  {
1944  // Get the next packet (if any)
1945  if (GetNextPacket() < 0)
1946  // Break loop when no more packets found
1947  break;
1948 
1949  // Video packet
1950  if (packet->stream_index == videoStream)
1951  {
1952  // Check if the AVFrame is finished and set it
1953  if (GetAVFrame())
1954  {
1955  // Update PTS / Frame Offset (if any)
1956  UpdatePTSOffset(true);
1957 
1958  // Get PTS of this packet
1959  pts = GetVideoPTS();
1960 
1961  // Remove pFrame
1962  RemoveAVFrame(pFrame);
1963 
1964  // Apply PTS offset
1965  pts += video_pts_offset;
1966 
1967  // Get the video packet start time (in seconds)
1968  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1969 
1970  // Increment the correct counter
1971  if (video_seconds <= 1.0)
1972  first_second_counter++;
1973  else if (video_seconds > 1.0 && video_seconds <= 2.0)
1974  second_second_counter++;
1975  else if (video_seconds > 2.0 && video_seconds <= 3.0)
1976  third_second_counter++;
1977  else if (video_seconds > 3.0 && video_seconds <= 4.0)
1978  forth_second_counter++;
1979  else if (video_seconds > 4.0 && video_seconds <= 5.0)
1980  fifth_second_counter++;
1981 
1982  // Increment counters
1983  frames_detected++;
1984  }
1985  }
1986  }
1987 
1988  // Double check that all counters have greater than zero (or give up)
1989  if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) {
1990  // Calculate average FPS (average of first few seconds)
1991  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1992  int avg_fps = round(sum_fps / 4.0f);
1993 
1994  // Update FPS
1995  info.fps = Fraction(avg_fps, 1);
1996 
1997  // Update Duration and Length
1998  info.video_length = frames_detected;
1999  info.duration = frames_detected / (sum_fps / 4.0f);
2000 
2001  // Update video bit rate
2003  } else if (second_second_counter != 0 && third_second_counter != 0) {
2004  // Calculate average FPS (only on second 2)
2005  int sum_fps = second_second_counter;
2006 
2007  // Update FPS
2008  info.fps = Fraction(sum_fps, 1);
2009 
2010  // Update Duration and Length
2011  info.video_length = frames_detected;
2012  info.duration = frames_detected / float(sum_fps);
2013 
2014  // Update video bit rate
2016  } else {
2017  // Too short to determine framerate, just default FPS
2018  // Set a few important default video settings (so audio can be divided into frames)
2019  info.fps.num = 30;
2020  info.fps.den = 1;
2021 
2022  // Calculate number of frames
2023  info.video_length = frames_detected;
2024  info.duration = frames_detected / info.fps.ToFloat();
2025  }
2026 }
2027 
2028 // Remove AVFrame from cache (and deallocate it's memory)
2029 void FFmpegReader::RemoveAVFrame(AVFrame* remove_frame)
2030 {
2031  // Remove pFrame (if exists)
2032  if (remove_frame)
2033  {
2034  // Free memory
2035  #pragma omp critical (packet_cache)
2036  {
2037  av_freep(&remove_frame->data[0]);
2038 #ifndef WIN32
2039  AV_FREE_FRAME(&remove_frame);
2040 #endif
2041  }
2042  }
2043 }
2044 
2045 // Remove AVPacket from cache (and deallocate it's memory)
2046 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
2047 {
2048  // deallocate memory for packet
2049  AV_FREE_PACKET(remove_packet);
2050 
2051  // Delete the object
2052  delete remove_packet;
2053 }
2054 
2055 /// Get the smallest video frame that is still being processed
2056 int64_t FFmpegReader::GetSmallestVideoFrame()
2057 {
2058  // Loop through frame numbers
2059  map<int64_t, int64_t>::iterator itr;
2060  int64_t smallest_frame = -1;
2061  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
2062  for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
2063  {
2064  if (itr->first < smallest_frame || smallest_frame == -1)
2065  smallest_frame = itr->first;
2066  }
2067 
2068  // Return frame number
2069  return smallest_frame;
2070 }
2071 
2072 /// Get the smallest audio frame that is still being processed
2073 int64_t FFmpegReader::GetSmallestAudioFrame()
2074 {
2075  // Loop through frame numbers
2076  map<int64_t, int64_t>::iterator itr;
2077  int64_t smallest_frame = -1;
2078  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
2079  for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
2080  {
2081  if (itr->first < smallest_frame || smallest_frame == -1)
2082  smallest_frame = itr->first;
2083  }
2084 
2085  // Return frame number
2086  return smallest_frame;
2087 }
2088 
2089 // Generate JSON string of this object
2091 
2092  // Return formatted string
2093  return JsonValue().toStyledString();
2094 }
2095 
2096 // Generate Json::JsonValue for this object
2098 
2099  // Create root json object
2100  Json::Value root = ReaderBase::JsonValue(); // get parent properties
2101  root["type"] = "FFmpegReader";
2102  root["path"] = path;
2103 
2104  // return JsonValue
2105  return root;
2106 }
2107 
2108 // Load JSON string into this object
2109 void FFmpegReader::SetJson(string value) {
2110 
2111  // Parse JSON string into JSON objects
2112  Json::Value root;
2113  Json::Reader reader;
2114  bool success = reader.parse( value, root );
2115  if (!success)
2116  // Raise exception
2117  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
2118 
2119  try
2120  {
2121  // Set all values that match
2122  SetJsonValue(root);
2123  }
2124  catch (exception e)
2125  {
2126  // Error parsing JSON (or missing keys)
2127  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
2128  }
2129 }
2130 
2131 // Load Json::JsonValue into this object
2132 void FFmpegReader::SetJsonValue(Json::Value root) {
2133 
2134  // Set parent data
2136 
2137  // Set data from Json (if key is found)
2138  if (!root["path"].isNull())
2139  path = root["path"].asString();
2140 
2141  // Re-Open path, and re-init everything (if needed)
2142  if (is_open)
2143  {
2144  Close();
2145  Open();
2146  }
2147 }
#define AV_RESET_FRAME(av_frame)
Point GetMaxPoint()
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:207
#define AV_FREE_FRAME(av_frame)
int MAX_HEIGHT
Maximum height for image data (useful for optimzing for a smaller preview or render) ...
Definition: Settings.h:92
int num
Numerator for the fraction.
Definition: Fraction.h:44
Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:215
#define SWRCONTEXT
#define AV_FIND_DECODER_CODEC_ID(av_stream)
CriticalSection processingCriticalSection
Definition: ReaderBase.h:102
#define SWR_INIT(ctx)
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:84
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:68
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
#define AV_REGISTER_ALL
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
float duration
Length of time (in seconds)
Definition: ReaderBase.h:65
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:80
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:46
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
Scale the clip until both height and width fill the canvas (cropping the overlap) ...
Definition: Enums.h:51
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define SWR_CLOSE(ctx)
#define SWR_FREE(ctx)
#define OPEN_MP_NUM_PROCESSORS
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
#define AVCODEC_REGISTER_ALL
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:62
void Close()
Close File.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: ReaderBase.h:73
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:66
FFmpegReader(string path)
std::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:81
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:63
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:100
#define AV_FREE_CONTEXT(av_context)
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
double Y
The Y value of the coordinate (usually representing the value of the property being animated) ...
Definition: Coordinate.h:57
int MAX_WIDTH
Maximum width for image data (useful for optimzing for a smaller preview or render) ...
Definition: Settings.h:89
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:85
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:75
#define FF_NUM_PROCESSORS
#define AV_FREE_PACKET(av_packet)
ScaleType scale
The scale determines how a clip should be resized to fit it's parent.
Definition: Clip.h:146
Exception when no streams are found in the file.
Definition: Exceptions.h:192
int height
The height of the video (in pixels)
Definition: ReaderBase.h:67
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
ClipBase * GetClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:252
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:114
auto AV_GET_CODEC_CONTEXT
#define PixelFormat
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:169
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:53
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:70
#define MY_INPUT_BUFFER_PADDING_SIZE
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:77
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
std::map< string, string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:87
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
int64_t Count()
Count the frames in the queue.
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: ReaderBase.h:72
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Coordinate co
This is the primary coordinate.
Definition: Point.h:83
Exception for invalid JSON.
Definition: Exceptions.h:152
#define AV_GET_CODEC_TYPE(av_stream)
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:69
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:61
Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:214
void Open()
Open File - which is called by the constructor automatically.
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: Settings.cpp:38
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:71
#define SWR_ALLOC()
#define PIX_FMT_RGBA
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:86
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:74
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:221
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:83
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:76
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:52
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:521
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:82