OpenShot Library | libopenshot  0.2.3
FFmpegWriter.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegWriter class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegWriter.h"
32 
33 using namespace openshot;
34 
36  path(path), fmt(NULL), oc(NULL), audio_st(NULL), video_st(NULL), audio_pts(0), video_pts(0), samples(NULL),
37  audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
38  initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(8), num_of_rescalers(32),
39  rescaler_position(0), video_codec(NULL), audio_codec(NULL), is_writing(false), write_video_count(0), write_audio_count(0),
40  original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
41  write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL)
42 {
43 
44  // Disable audio & video (so they can be independently enabled)
45  info.has_audio = false;
46  info.has_video = false;
47 
48  // Initialize FFMpeg, and register all formats and codecs
50 
51  // auto detect format
52  auto_detect_format();
53 }
54 
55 // Open the writer
57 {
58  if (!is_open) {
59  // Open the writer
60  is_open = true;
61 
62  // Prepare streams (if needed)
63  if (!prepare_streams)
65 
66  // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
67  if (info.has_video && video_st)
68  open_video(oc, video_st);
69  if (info.has_audio && audio_st)
70  open_audio(oc, audio_st);
71 
72  // Write header (if needed)
73  if (!write_header)
74  WriteHeader();
75  }
76 }
77 
78 // auto detect format (from path)
79 void FFmpegWriter::auto_detect_format()
80 {
81  // Auto detect the output format from the name. default is mpeg.
82  fmt = av_guess_format(NULL, path.c_str(), NULL);
83  if (!fmt)
84  throw InvalidFormat("Could not deduce output format from file extension.", path);
85 
86  // Allocate the output media context
87  AV_OUTPUT_CONTEXT(&oc, path.c_str());
88  if (!oc)
89  throw OutOfMemory("Could not allocate memory for AVFormatContext.", path);
90 
91  // Set the AVOutputFormat for the current AVFormatContext
92  oc->oformat = fmt;
93 
94  // Update codec names
95  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
96  // Update video codec name
97  info.vcodec = avcodec_find_encoder(fmt->video_codec)->name;
98 
99  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
100  // Update audio codec name
101  info.acodec = avcodec_find_encoder(fmt->audio_codec)->name;
102 }
103 
104 // initialize streams
105 void FFmpegWriter::initialize_streams()
106 {
107  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::initialize_streams", "fmt->video_codec", fmt->video_codec, "fmt->audio_codec", fmt->audio_codec, "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE, "", -1, "", -1, "", -1);
108 
109  // Add the audio and video streams using the default format codecs and initialize the codecs
110  video_st = NULL;
111  audio_st = NULL;
112  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
113  // Add video stream
114  video_st = add_video_stream();
115 
116  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
117  // Add audio stream
118  audio_st = add_audio_stream();
119 }
120 
121 // Set video export options
122 void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
123 {
124  // Set the video options
125  if (codec.length() > 0)
126  {
127  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
128  if (new_codec == NULL)
129  throw InvalidCodec("A valid video codec could not be found for this file.", path);
130  else {
131  // Set video codec
132  info.vcodec = new_codec->name;
133 
134  // Update video codec in fmt
135  fmt->video_codec = new_codec->id;
136  }
137  }
138  if (fps.num > 0)
139  {
140  // Set frames per second (if provided)
141  info.fps.num = fps.num;
142  info.fps.den = fps.den;
143 
144  // Set the timebase (inverse of fps)
147  }
148  if (width >= 1)
149  info.width = width;
150  if (height >= 1)
151  info.height = height;
152  if (pixel_ratio.num > 0)
153  {
154  info.pixel_ratio.num = pixel_ratio.num;
155  info.pixel_ratio.den = pixel_ratio.den;
156  }
157  if (bit_rate >= 1000) // bit_rate is the bitrate in b/s
158  info.video_bit_rate = bit_rate;
159  if ((bit_rate >= 0) && (bit_rate < 64) ) // bit_rate is the bitrate in crf
160  info.video_bit_rate = bit_rate;
161 
162  info.interlaced_frame = interlaced;
163  info.top_field_first = top_field_first;
164 
165  // Calculate the DAR (display aspect ratio)
167 
168  // Reduce size fraction
169  size.Reduce();
170 
171  // Set the ratio based on the reduced fraction
172  info.display_ratio.num = size.num;
173  info.display_ratio.den = size.den;
174 
175  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetVideoOptions (" + codec + ")", "width", width, "height", height, "size.num", size.num, "size.den", size.den, "fps.num", fps.num, "fps.den", fps.den);
176 
177  // Enable / Disable video
178  info.has_video = has_video;
179 }
180 
181 // Set audio export options
182 void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
183 {
184  // Set audio options
185  if (codec.length() > 0)
186  {
187  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
188  if (new_codec == NULL)
189  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
190  else
191  {
192  // Set audio codec
193  info.acodec = new_codec->name;
194 
195  // Update audio codec in fmt
196  fmt->audio_codec = new_codec->id;
197  }
198  }
199  if (sample_rate > 7999)
200  info.sample_rate = sample_rate;
201  if (channels > 0)
202  info.channels = channels;
203  if (bit_rate > 999)
204  info.audio_bit_rate = bit_rate;
205  info.channel_layout = channel_layout;
206 
207  // init resample options (if zero)
208  if (original_sample_rate == 0)
209  original_sample_rate = info.sample_rate;
210  if (original_channels == 0)
211  original_channels = info.channels;
212 
213  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetAudioOptions (" + codec + ")", "sample_rate", sample_rate, "channels", channels, "bit_rate", bit_rate, "", -1, "", -1, "", -1);
214 
215  // Enable / Disable audio
216  info.has_audio = has_audio;
217 }
218 
219 // Set custom options (some codecs accept additional params)
220 void FFmpegWriter::SetOption(StreamType stream, string name, string value)
221 {
222  // Declare codec context
223  AVCodecContext *c = NULL;
224  AVStream *st = NULL;
225  stringstream convert(value);
226 
227  if (info.has_video && stream == VIDEO_STREAM && video_st) {
228  st = video_st;
229  // Get codec context
230  c = AV_GET_CODEC_PAR_CONTEXT(st, video_codec);
231  }
232  else if (info.has_audio && stream == AUDIO_STREAM && audio_st) {
233  st = audio_st;
234  // Get codec context
235  c = AV_GET_CODEC_PAR_CONTEXT(st, audio_codec);
236  }
237  else
238  throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
239 
240  // Init AVOption
241  const AVOption *option = NULL;
242 
243  // Was a codec / stream found?
244  if (c)
245  // Find AVOption (if it exists)
246  option = AV_OPTION_FIND(c->priv_data, name.c_str());
247 
248  // Was option found?
249  if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
250  name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" ||
251  name == "crf"))
252  {
253  // Check for specific named options
254  if (name == "g")
255  // Set gop_size
256  convert >> c->gop_size;
257 
258  else if (name == "qmin")
259  // Minimum quantizer
260  convert >> c->qmin;
261 
262  else if (name == "qmax")
263  // Maximum quantizer
264  convert >> c->qmax;
265 
266  else if (name == "max_b_frames")
267  // Maximum number of B-frames between non-B-frames
268  convert >> c->max_b_frames;
269 
270  else if (name == "mb_decision")
271  // Macroblock decision mode
272  convert >> c->mb_decision;
273 
274  else if (name == "level")
275  // Set codec level
276  convert >> c->level;
277 
278  else if (name == "profile")
279  // Set codec profile
280  convert >> c->profile;
281 
282  else if (name == "slices")
283  // Indicates number of picture subdivisions
284  convert >> c->slices;
285 
286  else if (name == "rc_min_rate")
287  // Minimum bitrate
288  convert >> c->rc_min_rate;
289 
290  else if (name == "rc_max_rate")
291  // Maximum bitrate
292  convert >> c->rc_max_rate;
293 
294  else if (name == "rc_buffer_size")
295  // Buffer size
296  convert >> c->rc_buffer_size;
297 
298  else if (name == "crf") {
299  // encode quality and special settings like lossless
300  // This might be better in an extra methods as more options
301  // and way to set quality are possible
302  #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 39, 101)
303  switch (c->codec_id) {
304  #if (LIBAVCODEC_VERSION_MAJOR >= 58)
305  case AV_CODEC_ID_AV1 :
306  c->bit_rate = 0;
307  av_opt_set_int(c->priv_data, "crf", min(stoi(value),63), 0);
308  break;
309  #endif
310  case AV_CODEC_ID_VP8 :
311  c->bit_rate = 10000000;
312  av_opt_set_int(c->priv_data, "crf", max(min(stoi(value),63),4), 0); // 4-63
313  break;
314  case AV_CODEC_ID_VP9 :
315  c->bit_rate = 0; // Must be zero!
316  av_opt_set_int(c->priv_data, "crf", min(stoi(value),63), 0); // 0-63
317  if (stoi(value) == 0) {
318  av_opt_set(c->priv_data, "preset", "veryslow", 0);
319  av_opt_set_int(c->priv_data, "lossless", 1, 0);
320  }
321  break;
322  case AV_CODEC_ID_H264 :
323  av_opt_set_int(c->priv_data, "crf", min(stoi(value),51), 0); // 0-51
324  if (stoi(value) == 0) {
325  av_opt_set(c->priv_data, "preset", "veryslow", 0);
326  }
327  break;
328  case AV_CODEC_ID_H265 :
329  av_opt_set_int(c->priv_data, "crf", min(stoi(value),51), 0); // 0-51
330  if (stoi(value) == 0) {
331  av_opt_set(c->priv_data, "preset", "veryslow", 0);
332  av_opt_set_int(c->priv_data, "lossless", 1, 0);
333  }
334  break;
335  default:
336  // If this codec doesn't support crf calculate a bitrate
337  // TODO: find better formula
338  double mbs = 15000000.0;
339  if (info.video_bit_rate > 0) {
340  if (info.video_bit_rate > 42) {
341  mbs = 380.0;
342  }
343  else {
344  mbs *= pow(0.912,info.video_bit_rate);
345  }
346  }
347  c->bit_rate = (int)(mbs);
348  }
349  #endif
350  }
351 
352  else
353  // Set AVOption
354  AV_OPTION_SET(st, c->priv_data, name.c_str(), value.c_str(), c);
355 
356  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetOption (" + (string)name + ")", "stream == VIDEO_STREAM", stream == VIDEO_STREAM, "", -1, "", -1, "", -1, "", -1, "", -1);
357 
358  }
359  else
360  throw InvalidOptions("The option is not valid for this codec.", path);
361 
362 }
363 
364 /// Determine if codec name is valid
365 bool FFmpegWriter::IsValidCodec(string codec_name) {
366  // Initialize FFMpeg, and register all formats and codecs
368 
369  // Find the codec (if any)
370  if (avcodec_find_encoder_by_name(codec_name.c_str()) == NULL)
371  return false;
372  else
373  return true;
374 }
375 
376 // Prepare & initialize streams and open codecs
378 {
379  if (!info.has_audio && !info.has_video)
380  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
381 
382  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::PrepareStreams [" + path + "]", "info.has_audio", info.has_audio, "info.has_video", info.has_video, "", -1, "", -1, "", -1, "", -1);
383 
384  // Initialize the streams (i.e. add the streams)
385  initialize_streams();
386 
387  // Mark as 'prepared'
388  prepare_streams = true;
389 }
390 
391 // Write the file header (after the options are set)
393 {
394  if (!info.has_audio && !info.has_video)
395  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
396 
397  // Open the output file, if needed
398  if (!(fmt->flags & AVFMT_NOFILE)) {
399  if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
400  throw InvalidFile("Could not open or write file.", path);
401  }
402 
403  // Force the output filename (which doesn't always happen for some reason)
404  snprintf(oc->AV_FILENAME, sizeof(oc->AV_FILENAME), "%s", path.c_str());
405 
406  // Write the stream header, if any
407  // TODO: add avoptions / parameters instead of NULL
408 
409  // Add general metadata (if any)
410  for(std::map<string, string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter)
411  {
412  av_dict_set(&oc->metadata, iter->first.c_str(), iter->second.c_str(), 0);
413  }
414 
415  if (avformat_write_header(oc, NULL) != 0) {
416  throw InvalidFile("Could not write header to file.", path);
417  };
418 
419  // Mark as 'written'
420  write_header = true;
421 
422  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
423 }
424 
425 // Add a frame to the queue waiting to be encoded.
426 void FFmpegWriter::WriteFrame(std::shared_ptr<Frame> frame)
427 {
428  // Check for open reader (or throw exception)
429  if (!is_open)
430  throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
431 
432  // Add frame pointer to "queue", waiting to be processed the next
433  // time the WriteFrames() method is called.
434  if (info.has_video && video_st)
435  spooled_video_frames.push_back(frame);
436 
437  if (info.has_audio && audio_st)
438  spooled_audio_frames.push_back(frame);
439 
440  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame", "frame->number", frame->number, "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "cache_size", cache_size, "is_writing", is_writing, "", -1);
441 
442  // Write the frames once it reaches the correct cache size
443  if (spooled_video_frames.size() == cache_size || spooled_audio_frames.size() == cache_size)
444  {
445  // Is writer currently writing?
446  if (!is_writing)
447  // Write frames to video file
448  write_queued_frames();
449 
450  else
451  {
452  // Write frames to video file
453  write_queued_frames();
454  }
455  }
456 
457  // Keep track of the last frame added
458  last_frame = frame;
459 }
460 
461 // Write all frames in the queue to the video file.
462 void FFmpegWriter::write_queued_frames()
463 {
464  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_queued_frames", "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "", -1, "", -1, "", -1, "", -1);
465 
466  // Flip writing flag
467  is_writing = true;
468 
469  // Transfer spool to queue
470  queued_video_frames = spooled_video_frames;
471  queued_audio_frames = spooled_audio_frames;
472 
473  // Empty spool
474  spooled_video_frames.clear();
475  spooled_audio_frames.clear();
476 
477  // Set the number of threads in OpenMP
478  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
479  // Allow nested OpenMP sections
480  omp_set_nested(true);
481 
482  // Create blank exception
483  bool has_error_encoding_video = false;
484 
485  #pragma omp parallel
486  {
487  #pragma omp single
488  {
489  // Process all audio frames (in a separate thread)
490  if (info.has_audio && audio_st && !queued_audio_frames.empty())
491  write_audio_packets(false);
492 
493  // Loop through each queued image frame
494  while (!queued_video_frames.empty())
495  {
496  // Get front frame (from the queue)
497  std::shared_ptr<Frame> frame = queued_video_frames.front();
498 
499  // Add to processed queue
500  processed_frames.push_back(frame);
501 
502  // Encode and add the frame to the output file
503  if (info.has_video && video_st)
504  process_video_packet(frame);
505 
506  // Remove front item
507  queued_video_frames.pop_front();
508 
509  } // end while
510  } // end omp single
511 
512  #pragma omp single
513  {
514  // Loop back through the frames (in order), and write them to the video file
515  while (!processed_frames.empty())
516  {
517  // Get front frame (from the queue)
518  std::shared_ptr<Frame> frame = processed_frames.front();
519 
520  if (info.has_video && video_st)
521  {
522  // Add to deallocate queue (so we can remove the AVFrames when we are done)
523  deallocate_frames.push_back(frame);
524 
525  // Does this frame's AVFrame still exist
526  if (av_frames.count(frame))
527  {
528  // Get AVFrame
529  AVFrame *frame_final = av_frames[frame];
530 
531  // Write frame to video file
532  bool success = write_video_packet(frame, frame_final);
533  if (!success)
534  has_error_encoding_video = true;
535  }
536  }
537 
538  // Remove front item
539  processed_frames.pop_front();
540  }
541 
542  // Loop through, and deallocate AVFrames
543  while (!deallocate_frames.empty())
544  {
545  // Get front frame (from the queue)
546  std::shared_ptr<Frame> frame = deallocate_frames.front();
547 
548  // Does this frame's AVFrame still exist
549  if (av_frames.count(frame))
550  {
551  // Get AVFrame
552  AVFrame *av_frame = av_frames[frame];
553 
554  // Deallocate AVPicture and AVFrame
555  av_freep(&(av_frame->data[0]));
556  AV_FREE_FRAME(&av_frame);
557  av_frames.erase(frame);
558  }
559 
560  // Remove front item
561  deallocate_frames.pop_front();
562  }
563 
564  // Done writing
565  is_writing = false;
566 
567  } // end omp single
568  } // end omp parallel
569 
570  // Raise exception from main thread
571  if (has_error_encoding_video)
572  throw ErrorEncodingVideo("Error while writing raw video frame", -1);
573 }
574 
575 // Write a block of frames from a reader
576 void FFmpegWriter::WriteFrame(ReaderBase* reader, int64_t start, int64_t length)
577 {
578  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame (from Reader)", "start", start, "length", length, "", -1, "", -1, "", -1, "", -1);
579 
580  // Loop through each frame (and encoded it)
581  for (int64_t number = start; number <= length; number++)
582  {
583  // Get the frame
584  std::shared_ptr<Frame> f = reader->GetFrame(number);
585 
586  // Encode frame
587  WriteFrame(f);
588  }
589 }
590 
591 // Write the file trailer (after all frames are written)
593 {
594  // Write any remaining queued frames to video file
595  write_queued_frames();
596 
597  // Process final audio frame (if any)
598  if (info.has_audio && audio_st)
599  write_audio_packets(true);
600 
601  // Flush encoders (who sometimes hold on to frames)
602  flush_encoders();
603 
604  /* write the trailer, if any. The trailer must be written
605  * before you close the CodecContexts open when you wrote the
606  * header; otherwise write_trailer may try to use memory that
607  * was freed on av_codec_close() */
608  av_write_trailer(oc);
609 
610  // Mark as 'written'
611  write_trailer = true;
612 
613  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
614 }
615 
616 // Flush encoders
617 void FFmpegWriter::flush_encoders()
618 {
619  if (info.has_audio && audio_codec && AV_GET_CODEC_TYPE(audio_st) == AVMEDIA_TYPE_AUDIO && AV_GET_CODEC_ATTRIBUTES(audio_st, audio_codec)->frame_size <= 1)
620  return;
621 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
622  if (info.has_video && video_codec && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && (oc->oformat->flags & AVFMT_RAWPICTURE) && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
623  return;
624 #endif
625 
626  int error_code = 0;
627  int stop_encoding = 1;
628 
629  // FLUSH VIDEO ENCODER
630  if (info.has_video)
631  for (;;) {
632 
633  // Increment PTS (in frames and scaled to the codec's timebase)
634  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
635 
636  AVPacket pkt;
637  av_init_packet(&pkt);
638  pkt.data = NULL;
639  pkt.size = 0;
640 
641  // Pointer for video buffer (if using old FFmpeg version)
642  uint8_t *video_outbuf = NULL;
643 
644  /* encode the image */
645  int got_packet = 0;
646  int error_code = 0;
647 
648  #if IS_FFMPEG_3_2
649  #pragma omp critical (write_video_packet)
650  {
651  // Encode video packet (latest version of FFmpeg)
652  error_code = avcodec_send_frame(video_codec, NULL);
653  got_packet = 0;
654  while (error_code >= 0) {
655  error_code = avcodec_receive_packet(video_codec, &pkt);
656  if (error_code == AVERROR(EAGAIN)|| error_code == AVERROR_EOF) {
657  got_packet = 0;
658  // Write packet
659  avcodec_flush_buffers(video_codec);
660  break;
661  }
662  if (pkt.pts != AV_NOPTS_VALUE)
663  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
664  if (pkt.dts != AV_NOPTS_VALUE)
665  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
666  if (pkt.duration > 0)
667  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
668  pkt.stream_index = video_st->index;
669  error_code = av_interleaved_write_frame(oc, &pkt);
670  }
671  }
672  #else
673 
674  #if LIBAVFORMAT_VERSION_MAJOR >= 54
675  // Encode video packet (older than FFmpeg 3.2)
676  error_code = avcodec_encode_video2(video_codec, &pkt, NULL, &got_packet);
677 
678  #else
679  // Encode video packet (even older version of FFmpeg)
680  int video_outbuf_size = 0;
681 
682  /* encode the image */
683  int out_size = avcodec_encode_video(video_codec, NULL, video_outbuf_size, NULL);
684 
685  /* if zero size, it means the image was buffered */
686  if (out_size > 0) {
687  if(video_codec->coded_frame->key_frame)
688  pkt.flags |= AV_PKT_FLAG_KEY;
689  pkt.data= video_outbuf;
690  pkt.size= out_size;
691 
692  // got data back (so encode this frame)
693  got_packet = 1;
694  }
695  #endif
696  #endif
697 
698  if (error_code < 0) {
699  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
700  }
701  if (!got_packet) {
702  stop_encoding = 1;
703  break;
704  }
705 
706  // Override PTS (in frames and scaled to the codec's timebase)
707  //pkt.pts = write_video_count;
708 
709  // set the timestamp
710  if (pkt.pts != AV_NOPTS_VALUE)
711  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
712  if (pkt.dts != AV_NOPTS_VALUE)
713  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
714  if (pkt.duration > 0)
715  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
716  pkt.stream_index = video_st->index;
717 
718  // Write packet
719  error_code = av_interleaved_write_frame(oc, &pkt);
720  if (error_code < 0) {
721  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
722  }
723 
724  // Deallocate memory (if needed)
725  if (video_outbuf)
726  av_freep(&video_outbuf);
727  }
728 
729  // FLUSH AUDIO ENCODER
730  if (info.has_audio)
731  for (;;) {
732 
733  // Increment PTS (in samples and scaled to the codec's timebase)
734 #if LIBAVFORMAT_VERSION_MAJOR >= 54
735  // for some reason, it requires me to multiply channels X 2
736  write_audio_count += av_rescale_q(audio_input_position / (audio_codec->channels * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)), (AVRational){1, info.sample_rate}, audio_codec->time_base);
737 #else
738  write_audio_count += av_rescale_q(audio_input_position / audio_codec->channels, (AVRational){1, info.sample_rate}, audio_codec->time_base);
739 #endif
740 
741  AVPacket pkt;
742  av_init_packet(&pkt);
743  pkt.data = NULL;
744  pkt.size = 0;
745  pkt.pts = pkt.dts = write_audio_count;
746 
747  /* encode the image */
748  int got_packet = 0;
749  #if IS_FFMPEG_3_2
750  avcodec_send_frame(audio_codec, NULL);
751  got_packet = 0;
752  #else
753  error_code = avcodec_encode_audio2(audio_codec, &pkt, NULL, &got_packet);
754  #endif
755  if (error_code < 0) {
756  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
757  }
758  if (!got_packet) {
759  stop_encoding = 1;
760  break;
761  }
762 
763  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
764  // but it fixes lots of PTS related issues when I do this.
765  pkt.pts = pkt.dts = write_audio_count;
766 
767  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
768  if (pkt.pts != AV_NOPTS_VALUE)
769  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
770  if (pkt.dts != AV_NOPTS_VALUE)
771  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
772  if (pkt.duration > 0)
773  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
774 
775  // set stream
776  pkt.stream_index = audio_st->index;
777  pkt.flags |= AV_PKT_FLAG_KEY;
778 
779  // Write packet
780  error_code = av_interleaved_write_frame(oc, &pkt);
781  if (error_code < 0) {
782  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
783  }
784 
785  // deallocate memory for packet
786  AV_FREE_PACKET(&pkt);
787  }
788 
789 
790 }
791 
792 // Close the video codec
793 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
794 {
795  AV_FREE_CONTEXT(video_codec);
796  video_codec = NULL;
797 }
798 
799 // Close the audio codec
800 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
801 {
802  AV_FREE_CONTEXT(audio_codec);
803  audio_codec = NULL;
804 
805  // Clear buffers
806  delete[] samples;
807  delete[] audio_outbuf;
808  delete[] audio_encoder_buffer;
809  samples = NULL;
810  audio_outbuf = NULL;
811  audio_encoder_buffer = NULL;
812 
813  // Deallocate resample buffer
814  if (avr) {
815  SWR_CLOSE(avr);
816  SWR_FREE(&avr);
817  avr = NULL;
818  }
819 
820  if (avr_planar) {
821  SWR_CLOSE(avr_planar);
822  SWR_FREE(&avr_planar);
823  avr_planar = NULL;
824  }
825 }
826 
827 // Close the writer
829 {
830  // Write trailer (if needed)
831  if (!write_trailer)
832  WriteTrailer();
833 
834  // Close each codec
835  if (video_st)
836  close_video(oc, video_st);
837  if (audio_st)
838  close_audio(oc, audio_st);
839 
840  // Deallocate image scalers
841  if (image_rescalers.size() > 0)
842  RemoveScalers();
843 
844  // Free the streams
845  for (int i = 0; i < oc->nb_streams; i++) {
846  av_freep(AV_GET_CODEC_ATTRIBUTES(&oc->streams[i], &oc->streams[i]));
847  av_freep(&oc->streams[i]);
848  }
849 
850  if (!(fmt->flags & AVFMT_NOFILE)) {
851  /* close the output file */
852  avio_close(oc->pb);
853  }
854 
855  // Reset frame counters
856  write_video_count = 0;
857  write_audio_count = 0;
858 
859  // Free the context
860  av_freep(&oc);
861 
862  // Close writer
863  is_open = false;
864  prepare_streams = false;
865  write_header = false;
866  write_trailer = false;
867 
868  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
869 }
870 
871 // Add an AVFrame to the cache
872 void FFmpegWriter::add_avframe(std::shared_ptr<Frame> frame, AVFrame* av_frame)
873 {
874  // Add AVFrame to map (if it does not already exist)
875  if (!av_frames.count(frame))
876  {
877  // Add av_frame
878  av_frames[frame] = av_frame;
879  }
880  else
881  {
882  // Do not add, and deallocate this AVFrame
883  AV_FREE_FRAME(&av_frame);
884  }
885 }
886 
887 // Add an audio output stream
888 AVStream* FFmpegWriter::add_audio_stream()
889 {
890  AVCodecContext *c;
891  AVStream *st;
892 
893  // Find the audio codec
894  AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
895  if (codec == NULL)
896  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
897 
898  // Create a new audio stream
899  AV_FORMAT_NEW_STREAM(oc, audio_codec, codec, st)
900 
901  c->codec_id = codec->id;
902 #if LIBAVFORMAT_VERSION_MAJOR >= 53
903  c->codec_type = AVMEDIA_TYPE_AUDIO;
904 #else
905  c->codec_type = CODEC_TYPE_AUDIO;
906 #endif
907 
908  // Set the sample parameters
909  c->bit_rate = info.audio_bit_rate;
910  c->channels = info.channels;
911 
912  // Set valid sample rate (or throw error)
913  if (codec->supported_samplerates) {
914  int i;
915  for (i = 0; codec->supported_samplerates[i] != 0; i++)
916  if (info.sample_rate == codec->supported_samplerates[i])
917  {
918  // Set the valid sample rate
919  c->sample_rate = info.sample_rate;
920  break;
921  }
922  if (codec->supported_samplerates[i] == 0)
923  throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
924  } else
925  // Set sample rate
926  c->sample_rate = info.sample_rate;
927 
928 
929  // Set a valid number of channels (or throw error)
930  int channel_layout = info.channel_layout;
931  if (codec->channel_layouts) {
932  int i;
933  for (i = 0; codec->channel_layouts[i] != 0; i++)
934  if (channel_layout == codec->channel_layouts[i])
935  {
936  // Set valid channel layout
937  c->channel_layout = channel_layout;
938  break;
939  }
940  if (codec->channel_layouts[i] == 0)
941  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
942  } else
943  // Set valid channel layout
944  c->channel_layout = channel_layout;
945 
946  // Choose a valid sample_fmt
947  if (codec->sample_fmts) {
948  for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++)
949  {
950  // Set sample format to 1st valid format (and then exit loop)
951  c->sample_fmt = codec->sample_fmts[i];
952  break;
953  }
954  }
955  if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
956  // Default if no sample formats found
957  c->sample_fmt = AV_SAMPLE_FMT_S16;
958  }
959 
960  // some formats want stream headers to be separate
961  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
962 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
963  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
964 #else
965  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
966 #endif
967 
969  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_audio_stream", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->channels", c->channels, "c->sample_fmt", c->sample_fmt, "c->channel_layout", c->channel_layout, "c->sample_rate", c->sample_rate);
970 
971  return st;
972 }
973 
974 // Add a video output stream
975 AVStream* FFmpegWriter::add_video_stream()
976 {
977  AVCodecContext *c;
978  AVStream *st;
979 
980  // Find the video codec
981  AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
982  if (codec == NULL)
983  throw InvalidCodec("A valid video codec could not be found for this file.", path);
984 
985  // Create a new video stream
986  AV_FORMAT_NEW_STREAM(oc, video_codec, codec, st)
987 
988  c->codec_id = codec->id;
989 #if LIBAVFORMAT_VERSION_MAJOR >= 53
990  c->codec_type = AVMEDIA_TYPE_VIDEO;
991 #else
992  c->codec_type = CODEC_TYPE_VIDEO;
993 #endif
994 
995  /* Init video encoder options */
996  if (info.video_bit_rate >= 1000) {
997  c->bit_rate = info.video_bit_rate;
998  if (info.video_bit_rate >= 1500000) {
999  c->qmin = 2;
1000  c->qmax = 30;
1001  }
1002  // Here should be the setting for low fixed bitrate
1003  // Defaults are used because mpeg2 otherwise had problems
1004  }
1005  else {
1006  c->qmin = 0;
1007  c->qmax = 63;
1008  }
1009 
1010  //TODO: Implement variable bitrate feature (which actually works). This implementation throws
1011  //invalid bitrate errors and rc buffer underflow errors, etc...
1012  //c->rc_min_rate = info.video_bit_rate;
1013  //c->rc_max_rate = info.video_bit_rate;
1014  //c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
1015  //if ( !c->rc_initial_buffer_occupancy )
1016  // c->rc_initial_buffer_occupancy = c->rc_buffer_size * 3/4;
1017 
1018  /* resolution must be a multiple of two */
1019  // TODO: require /2 height and width
1020  c->width = info.width;
1021  c->height = info.height;
1022 
1023  /* time base: this is the fundamental unit of time (in seconds) in terms
1024  of which frame timestamps are represented. for fixed-fps content,
1025  timebase should be 1/framerate and timestamp increments should be
1026  identically 1. */
1027  c->time_base.num = info.video_timebase.num;
1028  c->time_base.den = info.video_timebase.den;
1029  #if LIBAVFORMAT_VERSION_MAJOR >= 56
1030  c->framerate = av_inv_q(c->time_base);
1031  #endif
1032  st->avg_frame_rate = av_inv_q(c->time_base);
1033  st->time_base.num = info.video_timebase.num;
1034  st->time_base.den = info.video_timebase.den;
1035 
1036  c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
1037  c->max_b_frames = 10;
1038  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
1039  /* just for testing, we also add B frames */
1040  c->max_b_frames = 2;
1041  if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
1042  /* Needed to avoid using macroblocks in which some coeffs overflow.
1043  This does not happen with normal video, it just happens here as
1044  the motion of the chroma plane does not match the luma plane. */
1045  c->mb_decision = 2;
1046  // some formats want stream headers to be separate
1047  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1048 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1049  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1050 #else
1051  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1052 #endif
1053 
1054  // Find all supported pixel formats for this codec
1055  const PixelFormat* supported_pixel_formats = codec->pix_fmts;
1056  while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
1057  // Assign the 1st valid pixel format (if one is missing)
1058  if (c->pix_fmt == PIX_FMT_NONE)
1059  c->pix_fmt = *supported_pixel_formats;
1060  ++supported_pixel_formats;
1061  }
1062 
1063  // Codec doesn't have any pix formats?
1064  if (c->pix_fmt == PIX_FMT_NONE) {
1065  if(fmt->video_codec == AV_CODEC_ID_RAWVIDEO) {
1066  // Raw video should use RGB24
1067  c->pix_fmt = PIX_FMT_RGB24;
1068 
1069 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
1070  if (strcmp(fmt->name, "gif") != 0)
1071  // If not GIF format, skip the encoding process
1072  // Set raw picture flag (so we don't encode this video)
1073  oc->oformat->flags |= AVFMT_RAWPICTURE;
1074 #endif
1075  } else {
1076  // Set the default codec
1077  c->pix_fmt = PIX_FMT_YUV420P;
1078  }
1079  }
1080 
1082 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
1083  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_video_stream (" + (string)fmt->name + " : " + (string)av_get_pix_fmt_name(c->pix_fmt) + ")", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->pix_fmt", c->pix_fmt, "oc->oformat->flags", oc->oformat->flags, "AVFMT_RAWPICTURE", AVFMT_RAWPICTURE, "", -1);
1084 #else
1085  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_video_stream (" + (string)fmt->name + " : " + (string)av_get_pix_fmt_name(c->pix_fmt) + ")", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->pix_fmt", c->pix_fmt, "oc->oformat->flags", oc->oformat->flags, "", -1, "", -1);
1086 #endif
1087 
1088  return st;
1089 }
1090 
1091 // open audio codec
1092 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st)
1093 {
1094  AVCodec *codec;
1095  AV_GET_CODEC_FROM_STREAM(st, audio_codec)
1096 
1097  // Set number of threads equal to number of processors (not to exceed 16)
1098  audio_codec->thread_count = min(FF_NUM_PROCESSORS, 16);
1099 
1100  // Find the audio encoder
1101  codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1102  if (!codec)
1103  codec = avcodec_find_encoder(audio_codec->codec_id);
1104  if (!codec)
1105  throw InvalidCodec("Could not find codec", path);
1106 
1107  // Init options
1108  AVDictionary *opts = NULL;
1109  av_dict_set(&opts, "strict", "experimental", 0);
1110 
1111  // Open the codec
1112  if (avcodec_open2(audio_codec, codec, &opts) < 0)
1113  throw InvalidCodec("Could not open codec", path);
1114  AV_COPY_PARAMS_FROM_CONTEXT(st, audio_codec);
1115 
1116  // Free options
1117  av_dict_free(&opts);
1118 
1119  // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
1120  // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
1121  if (audio_codec->frame_size <= 1) {
1122  // No frame size found... so calculate
1123  audio_input_frame_size = 50000 / info.channels;
1124 
1125  int s = AV_FIND_DECODER_CODEC_ID(st);
1126  switch (s) {
1127  case AV_CODEC_ID_PCM_S16LE:
1128  case AV_CODEC_ID_PCM_S16BE:
1129  case AV_CODEC_ID_PCM_U16LE:
1130  case AV_CODEC_ID_PCM_U16BE:
1131  audio_input_frame_size >>= 1;
1132  break;
1133  default:
1134  break;
1135  }
1136  } else {
1137  // Set frame size based on the codec
1138  audio_input_frame_size = audio_codec->frame_size;
1139  }
1140 
1141  // Set the initial frame size (since it might change during resampling)
1142  initial_audio_input_frame_size = audio_input_frame_size;
1143 
1144  // Allocate array for samples
1145  samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1146 
1147  // Set audio output buffer (used to store the encoded audio)
1148  audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1149  audio_outbuf = new uint8_t[audio_outbuf_size];
1150 
1151  // Set audio packet encoding buffer
1152  audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1153  audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1154 
1155  // Add audio metadata (if any)
1156  for(std::map<string, string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter)
1157  {
1158  av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1159  }
1160 
1161  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_audio", "audio_codec->thread_count", audio_codec->thread_count, "audio_input_frame_size", audio_input_frame_size, "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE, "", -1, "", -1, "", -1);
1162 
1163 }
1164 
1165 // open video codec
1166 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st)
1167 {
1168  AVCodec *codec;
1169  AV_GET_CODEC_FROM_STREAM(st, video_codec)
1170 
1171  // Set number of threads equal to number of processors (not to exceed 16)
1172  video_codec->thread_count = min(FF_NUM_PROCESSORS, 16);
1173 
1174  /* find the video encoder */
1175  codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1176  if (!codec)
1177  codec = avcodec_find_encoder(AV_FIND_DECODER_CODEC_ID(st));
1178  if (!codec)
1179  throw InvalidCodec("Could not find codec", path);
1180 
1181  /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */
1182  if(video_codec->max_b_frames && video_codec->codec_id != AV_CODEC_ID_MPEG4 && video_codec->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec->codec_id != AV_CODEC_ID_MPEG2VIDEO)
1183  video_codec->max_b_frames = 0;
1184 
1185  // Init options
1186  AVDictionary *opts = NULL;
1187  av_dict_set(&opts, "strict", "experimental", 0);
1188 
1189  /* open the codec */
1190  if (avcodec_open2(video_codec, codec, &opts) < 0)
1191  throw InvalidCodec("Could not open codec", path);
1192  AV_COPY_PARAMS_FROM_CONTEXT(st, video_codec);
1193 
1194  // Free options
1195  av_dict_free(&opts);
1196 
1197  // Add video metadata (if any)
1198  for(std::map<string, string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter)
1199  {
1200  av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1201  }
1202 
1203  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video", "video_codec->thread_count", video_codec->thread_count, "", -1, "", -1, "", -1, "", -1, "", -1);
1204 
1205 }
1206 
1207 // write all queued frames' audio to the video file
1208 void FFmpegWriter::write_audio_packets(bool final)
1209 {
1210  #pragma omp task firstprivate(final)
1211  {
1212  // Init audio buffers / variables
1213  int total_frame_samples = 0;
1214  int frame_position = 0;
1215  int channels_in_frame = 0;
1216  int sample_rate_in_frame = 0;
1217  int samples_in_frame = 0;
1218  ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1219 
1220  // Create a new array (to hold all S16 audio samples, for the current queued frames
1221  int16_t* all_queued_samples = (int16_t*)av_malloc((sizeof(int16_t)*(queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE)));
1222  int16_t* all_resampled_samples = NULL;
1223  int16_t* final_samples_planar = NULL;
1224  int16_t* final_samples = NULL;
1225 
1226  // Loop through each queued audio frame
1227  while (!queued_audio_frames.empty())
1228  {
1229  // Get front frame (from the queue)
1230  std::shared_ptr<Frame> frame = queued_audio_frames.front();
1231 
1232  // Get the audio details from this frame
1233  sample_rate_in_frame = frame->SampleRate();
1234  samples_in_frame = frame->GetAudioSamplesCount();
1235  channels_in_frame = frame->GetAudioChannelsCount();
1236  channel_layout_in_frame = frame->ChannelsLayout();
1237 
1238 
1239  // Get audio sample array
1240  float* frame_samples_float = NULL;
1241  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1242  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
1243 
1244 
1245  // Calculate total samples
1246  total_frame_samples = samples_in_frame * channels_in_frame;
1247 
1248  // Translate audio sample values back to 16 bit integers
1249  for (int s = 0; s < total_frame_samples; s++, frame_position++)
1250  // Translate sample value and copy into buffer
1251  all_queued_samples[frame_position] = int(frame_samples_float[s] * (1 << 15));
1252 
1253 
1254  // Deallocate float array
1255  delete[] frame_samples_float;
1256 
1257  // Remove front item
1258  queued_audio_frames.pop_front();
1259 
1260  } // end while
1261 
1262 
1263  // Update total samples (since we've combined all queued frames)
1264  total_frame_samples = frame_position;
1265  int remaining_frame_samples = total_frame_samples;
1266  int samples_position = 0;
1267 
1268 
1269  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets", "final", final, "total_frame_samples", total_frame_samples, "channel_layout_in_frame", channel_layout_in_frame, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "LAYOUT_MONO", LAYOUT_MONO);
1270 
1271  // Keep track of the original sample format
1272  AVSampleFormat output_sample_fmt = audio_codec->sample_fmt;
1273 
1274  AVFrame *audio_frame = NULL;
1275  if (!final) {
1276  // Create input frame (and allocate arrays)
1277  audio_frame = AV_ALLOCATE_FRAME();
1278  AV_RESET_FRAME(audio_frame);
1279  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1280 
1281  // Fill input frame with sample data
1282  avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples,
1283  audio_encoder_buffer_size, 0);
1284 
1285  // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1286  switch (audio_codec->sample_fmt)
1287  {
1288  case AV_SAMPLE_FMT_FLTP:
1289  {
1290  output_sample_fmt = AV_SAMPLE_FMT_FLT;
1291  break;
1292  }
1293  case AV_SAMPLE_FMT_S32P:
1294  {
1295  output_sample_fmt = AV_SAMPLE_FMT_S32;
1296  break;
1297  }
1298  case AV_SAMPLE_FMT_S16P:
1299  {
1300  output_sample_fmt = AV_SAMPLE_FMT_S16;
1301  break;
1302  }
1303  case AV_SAMPLE_FMT_U8P:
1304  {
1305  output_sample_fmt = AV_SAMPLE_FMT_U8;
1306  break;
1307  }
1308  }
1309 
1310  // Update total samples & input frame size (due to bigger or smaller data types)
1311  total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1312  total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1313 
1314  // Set remaining samples
1315  remaining_frame_samples = total_frame_samples;
1316 
1317  // Create output frame (and allocate arrays)
1318  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1319  AV_RESET_FRAME(audio_converted);
1320  audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1321  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1322 
1323  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (1st resampling)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", output_sample_fmt, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
1324 
1325  // setup resample context
1326  if (!avr) {
1327  avr = SWR_ALLOC();
1328  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1329  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1330  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1331  av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1332  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1333  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1334  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1335  av_opt_set_int(avr, "out_channels", info.channels, 0);
1336  SWR_INIT(avr);
1337  }
1338  int nb_samples = 0;
1339 
1340  // Convert audio samples
1341  nb_samples = SWR_CONVERT(avr, // audio resample context
1342  audio_converted->data, // output data pointers
1343  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1344  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1345  audio_frame->data, // input data pointers
1346  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1347  audio_frame->nb_samples); // number of input samples to convert
1348 
1349  // Create a new array (to hold all resampled S16 audio samples)
1350  all_resampled_samples = (int16_t*)av_malloc(sizeof(int16_t) * nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1351 
1352  // Copy audio samples over original samples
1353  memcpy(all_resampled_samples, audio_converted->data[0], nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1354 
1355  // Remove converted audio
1356  av_freep(&(audio_frame->data[0]));
1357  AV_FREE_FRAME(&audio_frame);
1358  av_freep(&audio_converted->data[0]);
1359  AV_FREE_FRAME(&audio_converted);
1360  all_queued_samples = NULL; // this array cleared with above call
1361 
1362  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)", "nb_samples", nb_samples, "remaining_frame_samples", remaining_frame_samples, "", -1, "", -1, "", -1, "", -1);
1363  }
1364 
1365  // Loop until no more samples
1366  while (remaining_frame_samples > 0 || final) {
1367  // Get remaining samples needed for this packet
1368  int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1369 
1370  // Determine how many samples we need
1371  int diff = 0;
1372  if (remaining_frame_samples >= remaining_packet_samples)
1373  diff = remaining_packet_samples;
1374  else if (remaining_frame_samples < remaining_packet_samples)
1375  diff = remaining_frame_samples;
1376 
1377  // Copy frame samples into the packet samples array
1378  if (!final)
1379  //TODO: Make this more sane
1380  memcpy(samples + (audio_input_position * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))), all_resampled_samples + samples_position, diff * av_get_bytes_per_sample(output_sample_fmt));
1381 
1382  // Increment counters
1383  audio_input_position += diff;
1384  samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1385  remaining_frame_samples -= diff;
1386  remaining_packet_samples -= diff;
1387 
1388  // Do we have enough samples to proceed?
1389  if (audio_input_position < (audio_input_frame_size * info.channels) && !final)
1390  // Not enough samples to encode... so wait until the next frame
1391  break;
1392 
1393  // Convert to planar (if needed by audio codec)
1394  AVFrame *frame_final = AV_ALLOCATE_FRAME();
1395  AV_RESET_FRAME(frame_final);
1396  if (av_sample_fmt_is_planar(audio_codec->sample_fmt))
1397  {
1398  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)", "in_sample_fmt", output_sample_fmt, "out_sample_fmt", audio_codec->sample_fmt, "in_sample_rate", info.sample_rate, "out_sample_rate", info.sample_rate, "in_channels", info.channels, "out_channels", info.channels);
1399 
1400  // setup resample context
1401  if (!avr_planar) {
1402  avr_planar = SWR_ALLOC();
1403  av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1404  av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1405  av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1406  av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec->sample_fmt, 0); // planar not allowed here
1407  av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1408  av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1409  av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1410  av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1411  SWR_INIT(avr_planar);
1412  }
1413 
1414  // Create input frame (and allocate arrays)
1415  audio_frame = AV_ALLOCATE_FRAME();
1416  AV_RESET_FRAME(audio_frame);
1417  audio_frame->nb_samples = audio_input_position / info.channels;
1418 
1419  // Create a new array
1420  final_samples_planar = (int16_t*)av_malloc(sizeof(int16_t) * audio_frame->nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1421 
1422  // Copy audio into buffer for frame
1423  memcpy(final_samples_planar, samples, audio_frame->nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1424 
1425  // Fill input frame with sample data
1426  avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt, (uint8_t *) final_samples_planar,
1427  audio_encoder_buffer_size, 0);
1428 
1429  // Create output frame (and allocate arrays)
1430  frame_final->nb_samples = audio_input_frame_size;
1431  av_samples_alloc(frame_final->data, frame_final->linesize, info.channels, frame_final->nb_samples, audio_codec->sample_fmt, 0);
1432 
1433  // Convert audio samples
1434  int nb_samples = SWR_CONVERT(avr_planar, // audio resample context
1435  frame_final->data, // output data pointers
1436  frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1437  frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1438  audio_frame->data, // input data pointers
1439  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1440  audio_frame->nb_samples); // number of input samples to convert
1441 
1442  // Copy audio samples over original samples
1443  if (nb_samples > 0)
1444  memcpy(samples, frame_final->data[0], nb_samples * av_get_bytes_per_sample(audio_codec->sample_fmt) * info.channels);
1445 
1446  // deallocate AVFrame
1447  av_freep(&(audio_frame->data[0]));
1448  AV_FREE_FRAME(&audio_frame);
1449  all_queued_samples = NULL; // this array cleared with above call
1450 
1451  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)", "nb_samples", nb_samples, "", -1, "", -1, "", -1, "", -1, "", -1);
1452 
1453  } else {
1454  // Create a new array
1455  final_samples = (int16_t*)av_malloc(sizeof(int16_t) * audio_input_position * (av_get_bytes_per_sample(audio_codec->sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1456 
1457  // Copy audio into buffer for frame
1458  memcpy(final_samples, samples, audio_input_position * av_get_bytes_per_sample(audio_codec->sample_fmt));
1459 
1460  // Init the nb_samples property
1461  frame_final->nb_samples = audio_input_frame_size;
1462 
1463  // Fill the final_frame AVFrame with audio (non planar)
1464  avcodec_fill_audio_frame(frame_final, audio_codec->channels, audio_codec->sample_fmt, (uint8_t *) final_samples,
1465  audio_encoder_buffer_size, 0);
1466  }
1467 
1468  // Increment PTS (in samples)
1469  write_audio_count += FFMIN(audio_input_frame_size, audio_input_position);
1470  frame_final->pts = write_audio_count; // Set the AVFrame's PTS
1471 
1472  // Init the packet
1473  AVPacket pkt;
1474  av_init_packet(&pkt);
1475  pkt.data = audio_encoder_buffer;
1476  pkt.size = audio_encoder_buffer_size;
1477 
1478  // Set the packet's PTS prior to encoding
1479  pkt.pts = pkt.dts = write_audio_count;
1480 
1481  /* encode the audio samples */
1482  int got_packet_ptr = 0;
1483 
1484  #if IS_FFMPEG_3_2
1485  // Encode audio (latest version of FFmpeg)
1486  int error_code;
1487  int ret = 0;
1488  int frame_finished = 0;
1489  error_code = ret = avcodec_send_frame(audio_codec, frame_final);
1490  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1491  avcodec_send_frame(audio_codec, NULL);
1492  }
1493  else {
1494  if (ret >= 0)
1495  pkt.size = 0;
1496  ret = avcodec_receive_packet(audio_codec, &pkt);
1497  if (ret >= 0)
1498  frame_finished = 1;
1499  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1500  avcodec_flush_buffers(audio_codec);
1501  ret = 0;
1502  }
1503  if (ret >= 0) {
1504  ret = frame_finished;
1505  }
1506  }
1507  if (!pkt.data && !frame_finished)
1508  {
1509  ret = -1;
1510  }
1511  got_packet_ptr = ret;
1512  #else
1513  // Encode audio (older versions of FFmpeg)
1514  int error_code = avcodec_encode_audio2(audio_codec, &pkt, frame_final, &got_packet_ptr);
1515  #endif
1516  /* if zero size, it means the image was buffered */
1517  if (error_code == 0 && got_packet_ptr) {
1518 
1519  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1520  // but it fixes lots of PTS related issues when I do this.
1521  pkt.pts = pkt.dts = write_audio_count;
1522 
1523  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1524  if (pkt.pts != AV_NOPTS_VALUE)
1525  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
1526  if (pkt.dts != AV_NOPTS_VALUE)
1527  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
1528  if (pkt.duration > 0)
1529  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
1530 
1531  // set stream
1532  pkt.stream_index = audio_st->index;
1533  pkt.flags |= AV_PKT_FLAG_KEY;
1534 
1535  /* write the compressed frame in the media file */
1536  int error_code = av_interleaved_write_frame(oc, &pkt);
1537  if (error_code < 0)
1538  {
1539  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1540  }
1541  }
1542 
1543  if (error_code < 0)
1544  {
1545  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1546  }
1547 
1548  // deallocate AVFrame
1549  av_freep(&(frame_final->data[0]));
1550  AV_FREE_FRAME(&frame_final);
1551 
1552  // deallocate memory for packet
1553  AV_FREE_PACKET(&pkt);
1554 
1555  // Reset position
1556  audio_input_position = 0;
1557  final = false;
1558  }
1559 
1560  // Delete arrays (if needed)
1561  if (all_resampled_samples) {
1562  av_freep(&all_resampled_samples);
1563  all_resampled_samples = NULL;
1564  }
1565  if (all_queued_samples) {
1566  av_freep(&all_queued_samples);
1567  all_queued_samples = NULL;
1568  }
1569 
1570  } // end task
1571 }
1572 
1573 // Allocate an AVFrame object
1574 AVFrame* FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer)
1575 {
1576  // Create an RGB AVFrame
1577  AVFrame *new_av_frame = NULL;
1578 
1579  // Allocate an AVFrame structure
1580  new_av_frame = AV_ALLOCATE_FRAME();
1581  if (new_av_frame == NULL)
1582  throw OutOfMemory("Could not allocate AVFrame", path);
1583 
1584  // Determine required buffer size and allocate buffer
1585  *buffer_size = AV_GET_IMAGE_SIZE(pix_fmt, width, height);
1586 
1587  // Create buffer (if not provided)
1588  if (!new_buffer)
1589  {
1590  // New Buffer
1591  new_buffer = (uint8_t*)av_malloc(*buffer_size * sizeof(uint8_t));
1592  // Attach buffer to AVFrame
1593  AV_COPY_PICTURE_DATA(new_av_frame, new_buffer, pix_fmt, width, height);
1594  new_av_frame->width = width;
1595  new_av_frame->height = height;
1596  new_av_frame->format = pix_fmt;
1597  }
1598 
1599  // return AVFrame
1600  return new_av_frame;
1601 }
1602 
1603 // process video frame
1604 void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame)
1605 {
1606  // Determine the height & width of the source image
1607  int source_image_width = frame->GetWidth();
1608  int source_image_height = frame->GetHeight();
1609 
1610  // Do nothing if size is 1x1 (i.e. no image in this frame)
1611  if (source_image_height == 1 && source_image_width == 1)
1612  return;
1613 
1614  // Init rescalers (if not initialized yet)
1615  if (image_rescalers.size() == 0)
1616  InitScalers(source_image_width, source_image_height);
1617 
1618  // Get a unique rescaler (for this thread)
1619  SwsContext *scaler = image_rescalers[rescaler_position];
1620  rescaler_position++;
1621  if (rescaler_position == num_of_rescalers)
1622  rescaler_position = 0;
1623 
1624  #pragma omp task firstprivate(frame, scaler, source_image_width, source_image_height)
1625  {
1626  // Allocate an RGB frame & final output frame
1627  int bytes_source = 0;
1628  int bytes_final = 0;
1629  AVFrame *frame_source = NULL;
1630  const uchar *pixels = NULL;
1631 
1632  // Get a list of pixels from source image
1633  pixels = frame->GetPixels();
1634 
1635  // Init AVFrame for source image & final (converted image)
1636  frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t*) pixels);
1637  #if IS_FFMPEG_3_2
1638  AVFrame *frame_final = allocate_avframe((AVPixelFormat)(video_st->codecpar->format), info.width, info.height, &bytes_final, NULL);
1639  #else
1640  AVFrame *frame_final = allocate_avframe(video_codec->pix_fmt, info.width, info.height, &bytes_final, NULL);
1641  #endif
1642 
1643  // Fill with data
1644  AV_COPY_PICTURE_DATA(frame_source, (uint8_t*)pixels, PIX_FMT_RGBA, source_image_width, source_image_height);
1645  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::process_video_packet", "frame->number", frame->number, "bytes_source", bytes_source, "bytes_final", bytes_final, "", -1, "", -1, "", -1);
1646 
1647  // Resize & convert pixel format
1648  sws_scale(scaler, frame_source->data, frame_source->linesize, 0,
1649  source_image_height, frame_final->data, frame_final->linesize);
1650 
1651  // Add resized AVFrame to av_frames map
1652  #pragma omp critical (av_frames_section)
1653  add_avframe(frame, frame_final);
1654 
1655  // Deallocate memory
1656  AV_FREE_FRAME(&frame_source);
1657 
1658  } // end task
1659 
1660 }
1661 
1662 // write video frame
1663 bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame* frame_final)
1664 {
1665 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
1666  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet", "frame->number", frame->number, "oc->oformat->flags", oc->oformat->flags, "", -1, "", -1, "", -1, "", -1);
1667 #else
1668  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet", "frame->number", frame->number, "oc->oformat->flags & AVFMT_RAWPICTURE", oc->oformat->flags & AVFMT_RAWPICTURE, "", -1, "", -1, "", -1, "", -1);
1669 
1670  if (oc->oformat->flags & AVFMT_RAWPICTURE) {
1671  // Raw video case.
1672  AVPacket pkt;
1673  av_init_packet(&pkt);
1674 
1675  pkt.flags |= AV_PKT_FLAG_KEY;
1676  pkt.stream_index= video_st->index;
1677  pkt.data= (uint8_t*)frame_final->data;
1678  pkt.size= sizeof(AVPicture);
1679 
1680  // Increment PTS (in frames and scaled to the codec's timebase)
1681  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1682  pkt.pts = write_video_count;
1683 
1684  /* write the compressed frame in the media file */
1685  int error_code = av_interleaved_write_frame(oc, &pkt);
1686  if (error_code < 0)
1687  {
1688  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1689  return false;
1690  }
1691 
1692  // Deallocate packet
1693  AV_FREE_PACKET(&pkt);
1694 
1695  } else
1696 #endif
1697  {
1698 
1699  AVPacket pkt;
1700  av_init_packet(&pkt);
1701  pkt.data = NULL;
1702  pkt.size = 0;
1703  pkt.pts = pkt.dts = AV_NOPTS_VALUE;
1704 
1705  // Pointer for video buffer (if using old FFmpeg version)
1706  uint8_t *video_outbuf = NULL;
1707 
1708  // Increment PTS (in frames and scaled to the codec's timebase)
1709  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1710 
1711  // Assign the initial AVFrame PTS from the frame counter
1712  frame_final->pts = write_video_count;
1713 
1714  /* encode the image */
1715  int got_packet_ptr = 0;
1716  int error_code = 0;
1717  #if IS_FFMPEG_3_2
1718  // Write video packet (latest version of FFmpeg)
1719  int frameFinished = 0;
1720  int ret = avcodec_send_frame(video_codec, frame_final);
1721  error_code = ret;
1722  if (ret < 0 ) {
1723  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet (Frame not sent)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
1724  if (ret == AVERROR(EAGAIN) )
1725  cerr << "Frame EAGAIN" << "\n";
1726  if (ret == AVERROR_EOF )
1727  cerr << "Frame AVERROR_EOF" << "\n";
1728  avcodec_send_frame(video_codec, NULL);
1729  }
1730  else {
1731  while (ret >= 0) {
1732  ret = avcodec_receive_packet(video_codec, &pkt);
1733  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1734  avcodec_flush_buffers(video_codec);
1735  got_packet_ptr = 0;
1736  break;
1737  }
1738  if (ret == 0) {
1739  got_packet_ptr = 1;
1740  break;
1741  }
1742  }
1743  }
1744  #else
1745  #if LIBAVFORMAT_VERSION_MAJOR >= 54
1746  // Write video packet (older than FFmpeg 3.2)
1747  error_code = avcodec_encode_video2(video_codec, &pkt, frame_final, &got_packet_ptr);
1748  if (error_code != 0 )
1749  cerr << "Frame AVERROR_EOF" << "\n";
1750  if (got_packet_ptr == 0 )
1751  cerr << "Frame gotpacket error" << "\n";
1752  #else
1753  // Write video packet (even older versions of FFmpeg)
1754  int video_outbuf_size = 200000;
1755  video_outbuf = (uint8_t*) av_malloc(200000);
1756 
1757  /* encode the image */
1758  int out_size = avcodec_encode_video(video_codec, video_outbuf, video_outbuf_size, frame_final);
1759 
1760  /* if zero size, it means the image was buffered */
1761  if (out_size > 0) {
1762  if(video_codec->coded_frame->key_frame)
1763  pkt.flags |= AV_PKT_FLAG_KEY;
1764  pkt.data= video_outbuf;
1765  pkt.size= out_size;
1766 
1767  // got data back (so encode this frame)
1768  got_packet_ptr = 1;
1769  }
1770  #endif
1771  #endif
1772 
1773  /* if zero size, it means the image was buffered */
1774  if (error_code == 0 && got_packet_ptr) {
1775 
1776  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1777  // but it fixes lots of PTS related issues when I do this.
1778  //pkt.pts = pkt.dts = write_video_count;
1779 
1780  // set the timestamp
1781  if (pkt.pts != AV_NOPTS_VALUE)
1782  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
1783  if (pkt.dts != AV_NOPTS_VALUE)
1784  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
1785  if (pkt.duration > 0)
1786  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
1787  pkt.stream_index = video_st->index;
1788 
1789  /* write the compressed frame in the media file */
1790  int error_code = av_interleaved_write_frame(oc, &pkt);
1791  if (error_code < 0)
1792  {
1793  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1794  return false;
1795  }
1796  }
1797 
1798  // Deallocate memory (if needed)
1799  if (video_outbuf)
1800  delete[] video_outbuf;
1801 
1802  // Deallocate packet
1803  AV_FREE_PACKET(&pkt);
1804  }
1805 
1806  // Success
1807  return true;
1808 }
1809 
1810 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
1812 {
1813  // output debug info
1814  av_dump_format(oc, 0, path.c_str(), 1);
1815 }
1816 
1817 // Init a collection of software rescalers (thread safe)
1818 void FFmpegWriter::InitScalers(int source_width, int source_height)
1819 {
1820  int scale_mode = SWS_FAST_BILINEAR;
1821  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1822  scale_mode = SWS_LANCZOS;
1823  }
1824 
1825  // Init software rescalers vector (many of them, one for each thread)
1826  for (int x = 0; x < num_of_rescalers; x++)
1827  {
1828  // Init the software scaler from FFMpeg
1829  img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(video_st, video_st->codec), scale_mode, NULL, NULL, NULL);
1830 
1831  // Add rescaler to vector
1832  image_rescalers.push_back(img_convert_ctx);
1833  }
1834 }
1835 
1836 // Set audio resample options
1837 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
1838  original_sample_rate = sample_rate;
1839  original_channels = channels;
1840 }
1841 
1842 // Remove & deallocate all software scalers
1844 {
1845  // Close all rescalers
1846  for (int x = 0; x < num_of_rescalers; x++)
1847  sws_freeContext(image_rescalers[x]);
1848 
1849  // Clear vector
1850  image_rescalers.clear();
1851 }
#define AV_RESET_FRAME(av_frame)
int channels
The number of audio channels used in the audio stream.
Definition: WriterBase.h:72
A video stream (used to determine which type of stream)
Definition: FFmpegWriter.h:65
#define AV_FREE_FRAME(av_frame)
void SetOption(StreamType stream, string name, string value)
Set custom options (some codecs accept additional params). This must be called after the PrepareStrea...
int num
Numerator for the fraction.
Definition: Fraction.h:44
WriterInfo info
Information about the current media file.
Definition: WriterBase.h:93
void OutputStreamInfo()
Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
#define AV_FIND_DECODER_CODEC_ID(av_stream)
An audio stream (used to determine which type of stream)
Definition: FFmpegWriter.h:66
#define SWR_INIT(ctx)
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: WriterBase.h:60
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: WriterBase.h:61
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:112
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
#define AV_REGISTER_ALL
std::map< string, string > metadata
An optional map/dictionary of video & audio metadata.
Definition: WriterBase.h:76
#define PIX_FMT_RGB24
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: WriterBase.h:69
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: WriterBase.h:63
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
#define AV_GET_CODEC_FROM_STREAM(av_stream, codec_in)
#define AV_OPTION_FIND(priv_data, name)
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define SWR_CLOSE(ctx)
#define SWR_FREE(ctx)
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:97
int width
The width of the video (in pixels)
Definition: WriterBase.h:57
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: WriterBase.h:70
#define OPEN_MP_NUM_PROCESSORS
void WriteFrame(std::shared_ptr< Frame > frame)
Add a frame to the stack waiting to be encoded.
Exception when encoding audio packet.
Definition: Exceptions.h:101
Exception when invalid sample rate is detected during encoding.
Definition: Exceptions.h:172
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: WriterBase.h:66
void Open()
Open writer.
void SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
Set video export options.
virtual std::shared_ptr< Frame > GetFrame(int64_t number)=0
#define AV_FREE_CONTEXT(av_context)
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
Exception when memory could not be allocated.
Definition: Exceptions.h:224
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Exception when invalid encoding options are used.
Definition: Exceptions.h:162
#define FF_NUM_PROCESSORS
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
Definition: Exceptions.h:192
void RemoveScalers()
Remove & deallocate all software scalers.
#define AV_ALLOCATE_FRAME()
bool top_field_first
Which interlaced field should be displayed first.
Definition: WriterBase.h:68
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
FFmpegWriter(string path)
Constructor for FFmpegWriter. Throws one of the following exceptions.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
void ResampleAudio(int sample_rate, int channels)
Set audio resample options.
#define PIX_FMT_YUV420P
static bool IsValidCodec(string codec_name)
Determine if codec name is valid.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: WriterBase.h:62
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
#define PixelFormat
void WriteTrailer()
Write the file trailer (after all frames are written). This is called automatically by the Close() me...
#define av_err2str(errnum)
void WriteHeader()
Write the file header (after the options are set). This method is called automatically by the Open() ...
void Close()
Close the writer.
bool interlaced_frame
Are the contents of this frame interlaced.
Definition: WriterBase.h:67
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: WriterBase.h:71
#define MY_INPUT_BUFFER_PADDING_SIZE
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
#define AV_OUTPUT_CONTEXT(output_context, path)
bool has_video
Determines if this file has a video stream.
Definition: WriterBase.h:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: WriterBase.h:59
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define AV_GET_CODEC_TYPE(av_stream)
#define AV_OPTION_SET(av_stream, priv_data, name, value, avcodec)
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: Settings.cpp:38
#define SWR_ALLOC()
bool has_audio
Determines if this file has an audio stream.
Definition: WriterBase.h:52
#define PIX_FMT_RGBA
void SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
Set audio export options.
Exception when a writer is closed, and a frame is requested.
Definition: Exceptions.h:264
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: WriterBase.h:73
void PrepareStreams()
Prepare & initialize streams and open codecs. This method is called automatically by the Open() metho...
int height
The height of the video (in pixels)
Definition: WriterBase.h:56
#define PIX_FMT_NONE
int den
Denominator for the fraction.
Definition: Fraction.h:45
#define AV_FORMAT_NEW_STREAM(oc, st_codec, av_codec, av_st)
Exception when no valid format is found for a file.
Definition: Exceptions.h:142
#define AUDIO_PACKET_ENCODING_SIZE
StreamType
This enumeration designates the type of stream when encoding (video or audio)
Definition: FFmpegWriter.h:63
#define AV_GET_CODEC_PAR_CONTEXT(av_stream, av_codec)
#define AV_COPY_PARAMS_FROM_CONTEXT(av_stream, av_codec)