31 #include "../include/FFmpegReader.h"
33 using namespace openshot;
36 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38 check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
58 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
59 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
60 check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
61 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
62 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
98 if (abs(diff) <= amount)
115 if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
116 throw InvalidFile(
"File could not be opened.", path);
119 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
125 for (
unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
128 if (
AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
132 if (
AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
136 if (videoStream == -1 && audioStream == -1)
137 throw NoStreamsFound(
"No video or audio streams found in this file.", path);
140 if (videoStream != -1)
146 pStream = pFormatCtx->streams[videoStream];
152 AVCodec *pCodec = avcodec_find_decoder(codecId);
158 if (pCodec == NULL) {
159 throw InvalidCodec(
"A valid video codec could not be found for this file.", path);
163 AVDictionary *opts = NULL;
164 av_dict_set(&opts,
"strict",
"experimental", 0);
167 if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
168 throw InvalidCodec(
"A video codec was found, but could not be opened.", path);
178 if (audioStream != -1)
184 aStream = pFormatCtx->streams[audioStream];
190 AVCodec *aCodec = avcodec_find_decoder(codecId);
196 if (aCodec == NULL) {
197 throw InvalidCodec(
"A valid audio codec could not be found for this file.", path);
201 AVDictionary *opts = NULL;
202 av_dict_set(&opts,
"strict",
"experimental", 0);
205 if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
206 throw InvalidCodec(
"An audio codec was found, but could not be opened.", path);
216 AVDictionaryEntry *tag = NULL;
217 while ((tag = av_dict_get(pFormatCtx->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
218 QString str_key = tag->key;
219 QString str_value = tag->value;
220 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
224 previous_packet_location.
frame = -1;
245 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
250 avcodec_flush_buffers(pCodecCtx);
255 avcodec_flush_buffers(aCodecCtx);
261 working_cache.
Clear();
262 missing_frames.
Clear();
267 processed_video_frames.clear();
268 processed_audio_frames.clear();
269 processing_video_frames.clear();
270 processing_audio_frames.clear();
271 missing_audio_frames.clear();
272 missing_video_frames.clear();
273 missing_audio_frames_source.clear();
274 missing_video_frames_source.clear();
275 checked_frames.clear();
279 avformat_close_input(&pFormatCtx);
280 av_freep(&pFormatCtx);
284 largest_frame_processed = 0;
285 seek_audio_frame_found = 0;
286 seek_video_frame_found = 0;
287 current_video_frame = 0;
288 has_missing_frames =
false;
292 void FFmpegReader::UpdateAudioInfo()
296 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
310 if (aStream->duration > 0.0f && aStream->duration >
info.
duration)
339 AVDictionaryEntry *tag = NULL;
340 while ((tag = av_dict_get(aStream->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
341 QString str_key = tag->key;
342 QString str_value = tag->value;
343 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
347 void FFmpegReader::UpdateVideoInfo()
355 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
365 if (pStream->sample_aspect_ratio.num != 0)
415 is_duration_known =
false;
420 is_duration_known =
true;
434 AVDictionaryEntry *tag = NULL;
435 while ((tag = av_dict_get(pStream->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
436 QString str_key = tag->key;
437 QString str_value = tag->value;
438 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
447 throw ReaderClosed(
"The FFmpegReader is closed. Call Open() before calling this method.", path);
450 if (requested_frame < 1)
456 throw InvalidFile(
"Could not detect the duration of the video or audio stream.", path);
459 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"requested_frame", requested_frame,
"last_frame", last_frame,
"", -1,
"", -1,
"", -1,
"", -1);
465 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
472 #pragma omp critical (ReadStream)
478 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame on 2nd look", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
488 if (last_frame == 0 && requested_frame != 1)
493 int64_t diff = requested_frame - last_frame;
494 if (diff >= 1 && diff <= 20)
497 frame = ReadStream(requested_frame);
504 Seek(requested_frame);
514 frame = ReadStream(requested_frame);
523 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
526 bool end_of_stream =
false;
527 bool check_seek =
false;
528 bool frame_finished =
false;
529 int packet_error = -1;
532 int packets_processed = 0;
534 int max_packets = 4096;
539 omp_set_nested(
true);
542 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream",
"requested_frame", requested_frame,
"OPEN_MP_NUM_PROCESSORS",
OPEN_MP_NUM_PROCESSORS,
"", -1,
"", -1,
"", -1,
"", -1);
552 packet_error = GetNextPacket();
554 int processing_video_frames_size = 0;
555 int processing_audio_frames_size = 0;
558 processing_video_frames_size = processing_video_frames.size();
559 processing_audio_frames_size = processing_audio_frames.size();
563 while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
566 processing_video_frames_size = processing_video_frames.size();
567 processing_audio_frames_size = processing_audio_frames.size();
571 if (packet_error < 0)
574 end_of_stream =
true;
579 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (GetNextPacket)",
"requested_frame", requested_frame,
"processing_video_frames_size", processing_video_frames_size,
"processing_audio_frames_size", processing_audio_frames_size,
"minimum_packets", minimum_packets,
"packets_processed", packets_processed,
"is_seeking", is_seeking);
585 num_packets_since_video_frame = 0;
589 #pragma omp critical (openshot_seek)
590 check_seek = CheckSeek(
true);
600 frame_finished = GetAVFrame();
606 UpdatePTSOffset(
true);
609 ProcessVideoPacket(requested_frame);
620 else if (
info.
has_audio && packet->stream_index == audioStream)
623 num_packets_since_video_frame++;
627 #pragma omp critical (openshot_seek)
628 check_seek = CheckSeek(
false);
638 UpdatePTSOffset(
false);
650 CheckWorkingFrames(
false, requested_frame);
660 if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
670 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (Completed)",
"packets_processed", packets_processed,
"end_of_stream", end_of_stream,
"largest_frame_processed", largest_frame_processed,
"Working Cache Count", working_cache.
Count(),
"", -1,
"", -1);
675 CheckWorkingFrames(end_of_stream, requested_frame);
692 std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
701 int FFmpegReader::GetNextPacket()
703 int found_packet = 0;
704 AVPacket *next_packet =
new AVPacket();
705 found_packet = av_read_frame(pFormatCtx, next_packet);
709 RemoveAVPacket(packet);
713 if (found_packet >= 0)
716 packet = next_packet;
724 bool FFmpegReader::GetAVFrame()
726 int frameFinished = -1;
731 #pragma omp critical (packet_cache)
735 ret = avcodec_send_packet(pCodecCtx, packet);
736 if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
737 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAVFrame (Packet not sent)",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
740 pFrame =
new AVFrame();
742 ret = avcodec_receive_frame(pCodecCtx, next_frame);
743 if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
748 if (frameFinished == 0 ) {
750 av_image_alloc(pFrame->data, pFrame->linesize,
info.
width,
info.
height, (AVPixelFormat)(pStream->codecpar->format), 1);
751 av_image_copy(pFrame->data, pFrame->linesize, (
const uint8_t**)next_frame->data, next_frame->linesize,
753 if (!check_interlace) {
754 check_interlace =
true;
762 avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
770 av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt,
info.
width,
774 if (!check_interlace) {
775 check_interlace =
true;
787 return frameFinished;
791 bool FFmpegReader::CheckSeek(
bool is_video)
798 if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
806 int64_t max_seeked_frame = seek_audio_frame_found;
807 if (seek_video_frame_found > max_seeked_frame)
808 max_seeked_frame = seek_video_frame_found;
811 if (max_seeked_frame >= seeking_frame)
814 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Too far, seek again)",
"is_video_seek", is_video_seek,
"max_seeked_frame", max_seeked_frame,
"seeking_frame", seeking_frame,
"seeking_pts", seeking_pts,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
817 Seek(seeking_frame - (10 * seek_count * seek_count));
822 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Successful)",
"is_video_seek", is_video_seek,
"current_pts", packet->pts,
"seeking_pts", seeking_pts,
"seeking_frame", seeking_frame,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
836 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame)
839 int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
842 if (!seek_video_frame_found && is_seeking)
843 seek_video_frame_found = current_frame;
846 if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
849 RemoveAVFrame(pFrame);
852 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Skipped)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
859 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Before)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
866 AVFrame *my_frame = pFrame;
870 processing_video_frames[current_frame] = current_frame;
872 #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt)
875 AVFrame *pFrameRGB = NULL;
877 uint8_t *buffer = NULL;
881 if (pFrameRGB == NULL)
882 throw OutOfBoundsFrame(
"Convert Image Broke!", current_frame, video_length);
902 max_width = max(
float(max_width), max_width * max_scale_x);
903 max_height = max(
float(max_height), max_height * max_scale_y);
909 QSize width_size(max_width * max_scale_x,
912 max_height * max_scale_y);
914 if (width_size.width() >= max_width && width_size.height() >= max_height) {
915 max_width = max(max_width, width_size.width());
916 max_height = max(max_height, width_size.height());
919 max_width = max(max_width, height_size.width());
920 max_height = max(max_height, height_size.height());
931 int original_height = height;
932 if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
934 float ratio = float(width) / float(height);
935 int possible_width = round(max_height * ratio);
936 int possible_height = round(max_width / ratio);
938 if (possible_width <= max_width) {
940 width = possible_width;
945 height = possible_height;
952 #pragma omp critical (video_buffer)
953 buffer = (uint8_t *) av_malloc(numBytes *
sizeof(uint8_t));
958 int scale_mode = SWS_FAST_BILINEAR;
960 scale_mode = SWS_LANCZOS;
966 sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
967 original_height, pFrameRGB->data, pFrameRGB->linesize);
970 std::shared_ptr<Frame> f = CreateFrame(current_frame);
973 f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
976 working_cache.
Add(f);
979 #pragma omp critical (video_buffer)
980 last_video_frame = f;
987 RemoveAVFrame(my_frame);
988 sws_freeContext(img_convert_ctx);
993 processing_video_frames.erase(current_frame);
994 processed_video_frames[current_frame] = current_frame;
998 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (After)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"f->number", f->number,
"", -1,
"", -1,
"", -1);
1005 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame,
int starting_sample)
1008 if (!seek_audio_frame_found && is_seeking)
1009 seek_audio_frame_found = target_frame;
1012 if (target_frame < (requested_frame - 20))
1015 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Skipped)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
1022 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Before)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
1025 int frame_finished = 0;
1029 int packet_samples = 0;
1034 #pragma omp critical (ProcessAudioPacket)
1039 while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
1041 ret = avcodec_send_packet(aCodecCtx, packet);
1042 if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1043 avcodec_send_packet(aCodecCtx, NULL);
1048 ret = avcodec_receive_frame(aCodecCtx, audio_frame);
1051 if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1052 avcodec_flush_buffers(aCodecCtx);
1056 ret = frame_finished;
1059 if (!packet->data && !frame_finished)
1064 int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1068 if (frame_finished) {
1072 int plane_size = -1;
1073 data_size = av_samples_get_buffer_size(&plane_size,
1075 audio_frame->nb_samples,
1083 int pts_remaining_samples = packet_samples /
info.
channels;
1086 int64_t adjusted_pts = packet->pts + audio_pts_offset;
1091 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info A)",
"pts_counter", pts_counter,
"PTS", adjusted_pts,
"Offset", audio_pts_offset,
"PTS Diff", adjusted_pts - prev_pts,
"Samples", pts_remaining_samples,
"Sample PTS ratio",
float(adjusted_pts - prev_pts) / pts_remaining_samples);
1092 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info B)",
"Sample Diff", pts_remaining_samples - prev_samples - prev_pts,
"Total", pts_total,
"PTS Seconds", audio_seconds,
"Sample Seconds", sample_seconds,
"Seconds Diff", audio_seconds - sample_seconds,
"raw samples", packet_samples);
1095 prev_pts = adjusted_pts;
1096 pts_total += pts_remaining_samples;
1098 prev_samples = pts_remaining_samples;
1103 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
1106 while (pts_remaining_samples)
1112 int samples = samples_per_frame - previous_packet_location.
sample_start;
1113 if (samples > pts_remaining_samples)
1114 samples = pts_remaining_samples;
1117 pts_remaining_samples -= samples;
1119 if (pts_remaining_samples > 0) {
1121 previous_packet_location.
frame++;
1127 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
1140 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (ReSample)",
"packet_samples", packet_samples,
"info.channels",
info.
channels,
"info.sample_rate",
info.
sample_rate,
"aCodecCtx->sample_fmt",
AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx),
"AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16,
"", -1);
1145 audio_converted->nb_samples = audio_frame->nb_samples;
1146 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1156 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1165 audio_converted->data,
1166 audio_converted->linesize[0],
1167 audio_converted->nb_samples,
1169 audio_frame->linesize[0],
1170 audio_frame->nb_samples);
1173 memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels);
1181 av_free(audio_converted->data[0]);
1184 int64_t starting_frame_number = -1;
1185 bool partial_frame =
true;
1186 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
1189 starting_frame_number = target_frame;
1190 int channel_buffer_size = packet_samples /
info.
channels;
1191 float *channel_buffer =
new float[channel_buffer_size];
1194 for (
int z = 0; z < channel_buffer_size; z++)
1195 channel_buffer[z] = 0.0f;
1201 for (
int sample = 0; sample < packet_samples; sample++)
1204 if (channel_filter == channel)
1207 channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1223 int start = starting_sample;
1224 int remaining_samples = channel_buffer_size;
1225 float *iterate_channel_buffer = channel_buffer;
1226 while (remaining_samples > 0)
1232 int samples = samples_per_frame - start;
1233 if (samples > remaining_samples)
1234 samples = remaining_samples;
1237 std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1240 if (samples_per_frame == start + samples)
1241 partial_frame =
false;
1243 partial_frame =
true;
1247 f->AddAudio(
true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1250 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (f->AddAudio)",
"frame", starting_frame_number,
"start", start,
"samples", samples,
"channel", channel_filter,
"partial_frame", partial_frame,
"samples_per_frame", samples_per_frame);
1253 working_cache.
Add(f);
1256 remaining_samples -= samples;
1259 if (remaining_samples > 0)
1260 iterate_channel_buffer += samples;
1263 starting_frame_number++;
1270 delete[] channel_buffer;
1271 channel_buffer = NULL;
1272 iterate_channel_buffer = NULL;
1283 for (int64_t f = target_frame; f < starting_frame_number; f++) {
1287 processing_audio_frames.erase(processing_audio_frames.find(f));
1290 if (processing_audio_frames.count(f) == 0)
1292 processed_audio_frames[f] = f;
1295 if (target_frame == starting_frame_number) {
1297 processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1305 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (After)",
"requested_frame", requested_frame,
"starting_frame", target_frame,
"end_frame", starting_frame_number - 1,
"", -1,
"", -1,
"", -1);
1312 void FFmpegReader::Seek(int64_t requested_frame)
1315 if (requested_frame < 1)
1316 requested_frame = 1;
1320 int processing_video_frames_size = 0;
1321 int processing_audio_frames_size = 0;
1324 processing_video_frames_size = processing_video_frames.size();
1325 processing_audio_frames_size = processing_audio_frames.size();
1329 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Seek",
"requested_frame", requested_frame,
"seek_count", seek_count,
"last_frame", last_frame,
"processing_video_frames_size", processing_video_frames_size,
"processing_audio_frames_size", processing_audio_frames_size,
"video_pts_offset", video_pts_offset);
1332 while (processing_video_frames_size + processing_audio_frames_size > 0) {
1335 processing_video_frames_size = processing_video_frames.size();
1336 processing_audio_frames_size = processing_audio_frames.size();
1340 working_cache.
Clear();
1341 missing_frames.
Clear();
1346 processing_audio_frames.clear();
1347 processing_video_frames.clear();
1348 processed_video_frames.clear();
1349 processed_audio_frames.clear();
1350 missing_audio_frames.clear();
1351 missing_video_frames.clear();
1352 missing_audio_frames_source.clear();
1353 missing_video_frames_source.clear();
1354 checked_frames.clear();
1359 current_video_frame = 0;
1360 largest_frame_processed = 0;
1361 num_checks_since_final = 0;
1362 num_packets_since_video_frame = 0;
1363 has_missing_frames =
false;
1372 if (requested_frame - buffer_amount < 20)
1384 if (seek_count == 1) {
1387 seeking_pts = ConvertFrameToVideoPTS(1);
1389 seek_audio_frame_found = 0;
1390 seek_video_frame_found = 0;
1395 bool seek_worked =
false;
1396 int64_t seek_target = 0;
1401 seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1403 fprintf(stderr,
"%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1407 is_video_seek =
true;
1415 seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1417 fprintf(stderr,
"%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1421 is_video_seek =
false;
1431 avcodec_flush_buffers(aCodecCtx);
1435 avcodec_flush_buffers(pCodecCtx);
1438 previous_packet_location.
frame = -1;
1443 if (seek_count == 1) {
1445 seeking_pts = seek_target;
1446 seeking_frame = requested_frame;
1448 seek_audio_frame_found = 0;
1449 seek_video_frame_found = 0;
1475 int64_t FFmpegReader::GetVideoPTS()
1477 int64_t current_pts = 0;
1478 if(packet->dts != AV_NOPTS_VALUE)
1479 current_pts = packet->dts;
1486 void FFmpegReader::UpdatePTSOffset(
bool is_video)
1492 if (video_pts_offset == 99999)
1498 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::UpdatePTSOffset (Video)",
"video_pts_offset", video_pts_offset,
"is_video", is_video,
"", -1,
"", -1,
"", -1,
"", -1);
1504 if (audio_pts_offset == 99999)
1510 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::UpdatePTSOffset (Audio)",
"audio_pts_offset", audio_pts_offset,
"is_video", is_video,
"", -1,
"", -1,
"", -1,
"", -1);
1516 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts)
1519 pts = pts + video_pts_offset;
1520 int64_t previous_video_frame = current_video_frame;
1529 if (current_video_frame == 0)
1530 current_video_frame = frame;
1534 if (frame == previous_video_frame) {
1540 current_video_frame++;
1542 if (current_video_frame < frame)
1544 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)",
"calculated frame", frame,
"previous_video_frame", previous_video_frame,
"current_video_frame", current_video_frame,
"", -1,
"", -1,
"", -1);
1549 while (current_video_frame < frame) {
1550 if (!missing_video_frames.count(current_video_frame)) {
1551 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)",
"current_video_frame", current_video_frame,
"previous_video_frame", previous_video_frame,
"", -1,
"", -1,
"", -1,
"", -1);
1552 missing_video_frames.insert(pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1553 missing_video_frames_source.insert(pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1557 has_missing_frames =
true;
1560 current_video_frame++;
1569 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number)
1578 return video_pts - video_pts_offset;
1582 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number)
1591 return audio_pts - audio_pts_offset;
1595 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
1598 pts = pts + audio_pts_offset;
1607 int64_t whole_frame = int64_t(frame);
1610 double sample_start_percentage = frame - double(whole_frame);
1616 int sample_start = round(
double(samples_per_frame) * sample_start_percentage);
1619 if (whole_frame < 1)
1621 if (sample_start < 0)
1628 if (previous_packet_location.
frame != -1) {
1629 if (location.
is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1631 int64_t orig_frame = location.
frame;
1636 location.
frame = previous_packet_location.
frame;
1639 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)",
"Source Frame", orig_frame,
"Source Audio Sample", orig_start,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1);
1643 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)",
"Previous location frame", previous_packet_location.
frame,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1,
"", -1);
1646 for (int64_t audio_frame = previous_packet_location.
frame; audio_frame < location.
frame; audio_frame++) {
1647 if (!missing_audio_frames.count(audio_frame)) {
1648 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (tracking missing frame)",
"missing_audio_frame", audio_frame,
"previous_audio_frame", previous_packet_location.
frame,
"new location frame", location.
frame,
"", -1,
"", -1,
"", -1);
1649 missing_audio_frames.insert(pair<int64_t, int64_t>(audio_frame, previous_packet_location.
frame - 1));
1656 previous_packet_location = location;
1663 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame)
1666 std::shared_ptr<Frame> output = working_cache.
GetFrame(requested_frame);
1675 working_cache.
Add(output);
1678 if (requested_frame > largest_frame_processed)
1679 largest_frame_processed = requested_frame;
1687 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
1690 bool seek_trash =
false;
1691 int64_t max_seeked_frame = seek_audio_frame_found;
1692 if (seek_video_frame_found > max_seeked_frame)
1693 max_seeked_frame = seek_video_frame_found;
1694 if ((
info.
has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1695 (
info.
has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1702 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
1708 int checked_count = 0;
1711 if (checked_frames.count(requested_frame) == 0)
1712 checked_frames[requested_frame] = 1;
1714 checked_frames[requested_frame]++;
1715 checked_count = checked_frames[requested_frame];
1718 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame",
"requested_frame", requested_frame,
"has_missing_frames", has_missing_frames,
"missing_video_frames.size()", missing_video_frames.size(),
"checked_count", checked_count,
"", -1,
"", -1);
1721 map<int64_t, int64_t>::iterator itr;
1722 bool found_missing_frame =
false;
1731 if (checked_count > 8 && !missing_video_frames.count(requested_frame) &&
1732 !processing_audio_frames.count(requested_frame) && processed_audio_frames.count(requested_frame) &&
1733 last_frame && last_video_frame->has_image_data && aCodecId == AV_CODEC_ID_MP3 && (vCodecId == AV_CODEC_ID_MJPEGB || vCodecId == AV_CODEC_ID_MJPEG)) {
1734 missing_video_frames.insert(pair<int64_t, int64_t>(requested_frame, last_video_frame->number));
1735 missing_video_frames_source.insert(pair<int64_t, int64_t>(last_video_frame->number, requested_frame));
1736 missing_frames.
Add(last_video_frame);
1741 if (missing_video_frames.count(requested_frame)) {
1742 int64_t missing_source_frame = missing_video_frames.find(requested_frame)->second;
1745 if (checked_frames.count(missing_source_frame) == 0)
1746 checked_frames[missing_source_frame] = 1;
1748 checked_frames[missing_source_frame]++;
1751 std::shared_ptr<Frame> parent_frame = missing_frames.
GetFrame(missing_source_frame);
1752 if (parent_frame == NULL) {
1754 if (parent_frame != NULL) {
1756 missing_frames.
Add(parent_frame);
1761 std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1764 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1767 if (parent_frame != NULL) {
1769 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1772 std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
1774 missing_frame->AddImage(std::shared_ptr<QImage>(
new QImage(*parent_image)));
1775 processed_video_frames[missing_frame->number] = missing_frame->number;
1781 if (missing_audio_frames.count(requested_frame)) {
1784 std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1790 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (Add Silence for Missing Audio Frame)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"samples_per_frame", samples_per_frame,
"", -1,
"", -1,
"", -1);
1793 missing_frame->AddAudioSilence(samples_per_frame);
1794 processed_audio_frames[missing_frame->number] = missing_frame->number;
1797 return found_missing_frame;
1801 void FFmpegReader::CheckWorkingFrames(
bool end_of_stream, int64_t requested_frame)
1804 bool checked_count_tripped =
false;
1805 int max_checked_count = 80;
1808 CheckMissingFrame(requested_frame);
1822 working_cache.
Remove(f->number);
1826 CheckMissingFrame(f->number);
1829 int checked_count = 0;
1830 int checked_frames_size = 0;
1832 bool is_video_ready =
false;
1833 bool is_audio_ready =
false;
1836 is_video_ready = processed_video_frames.count(f->number);
1837 is_audio_ready = processed_audio_frames.count(f->number);
1840 checked_frames_size = checked_frames.size();
1841 if (!checked_count_tripped || f->number >= requested_frame)
1842 checked_count = checked_frames[f->number];
1845 checked_count = max_checked_count;
1848 if (previous_packet_location.
frame == f->number && !end_of_stream)
1849 is_audio_ready =
false;
1850 bool is_seek_trash = IsPartialFrame(f->number);
1857 if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
1859 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (exceeded checked_count)",
"requested_frame", requested_frame,
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames_size", checked_frames_size);
1862 checked_count_tripped =
true;
1864 if (
info.
has_video && !is_video_ready && last_video_frame) {
1866 f->AddImage(std::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1867 is_video_ready =
true;
1872 is_audio_ready =
true;
1877 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames",
"requested_frame", requested_frame,
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames_size", checked_frames_size);
1880 if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash)
1883 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (mark frame as final)",
"requested_frame", requested_frame,
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"end_of_stream", end_of_stream);
1890 f->AddImage(std::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1893 num_checks_since_final = 0;
1901 if (missing_video_frames_source.count(f->number)) {
1903 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (add frame to missing cache)",
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Missing Cache Count", missing_frames.
Count(),
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1);
1904 missing_frames.
Add(f);
1908 checked_frames.erase(f->number);
1912 working_cache.
Remove(f->number);
1915 last_frame = f->number;
1919 working_cache.
Remove(f->number);
1929 void FFmpegReader::CheckFPS()
1933 int first_second_counter = 0;
1934 int second_second_counter = 0;
1935 int third_second_counter = 0;
1936 int forth_second_counter = 0;
1937 int fifth_second_counter = 0;
1938 int frames_detected = 0;
1945 if (GetNextPacket() < 0)
1950 if (packet->stream_index == videoStream)
1956 UpdatePTSOffset(
true);
1959 pts = GetVideoPTS();
1962 RemoveAVFrame(pFrame);
1965 pts += video_pts_offset;
1971 if (video_seconds <= 1.0)
1972 first_second_counter++;
1973 else if (video_seconds > 1.0 && video_seconds <= 2.0)
1974 second_second_counter++;
1975 else if (video_seconds > 2.0 && video_seconds <= 3.0)
1976 third_second_counter++;
1977 else if (video_seconds > 3.0 && video_seconds <= 4.0)
1978 forth_second_counter++;
1979 else if (video_seconds > 4.0 && video_seconds <= 5.0)
1980 fifth_second_counter++;
1989 if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) {
1991 int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1992 int avg_fps = round(sum_fps / 4.0f);
2003 }
else if (second_second_counter != 0 && third_second_counter != 0) {
2005 int sum_fps = second_second_counter;
2029 void FFmpegReader::RemoveAVFrame(AVFrame* remove_frame)
2035 #pragma omp critical (packet_cache)
2037 av_freep(&remove_frame->data[0]);
2046 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
2052 delete remove_packet;
2056 int64_t FFmpegReader::GetSmallestVideoFrame()
2059 map<int64_t, int64_t>::iterator itr;
2060 int64_t smallest_frame = -1;
2062 for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
2064 if (itr->first < smallest_frame || smallest_frame == -1)
2065 smallest_frame = itr->first;
2069 return smallest_frame;
2073 int64_t FFmpegReader::GetSmallestAudioFrame()
2076 map<int64_t, int64_t>::iterator itr;
2077 int64_t smallest_frame = -1;
2079 for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
2081 if (itr->first < smallest_frame || smallest_frame == -1)
2082 smallest_frame = itr->first;
2086 return smallest_frame;
2101 root[
"type"] =
"FFmpegReader";
2102 root[
"path"] = path;
2113 Json::Reader reader;
2114 bool success = reader.parse( value, root );
2117 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
2127 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
2138 if (!root[
"path"].isNull())
2139 path = root[
"path"].asString();
#define AV_RESET_FRAME(av_frame)
Point GetMaxPoint()
Get max point (by Y coordinate)
#define AV_FREE_FRAME(av_frame)
int MAX_HEIGHT
Maximum height for image data (useful for optimzing for a smaller preview or render) ...
int num
Numerator for the fraction.
Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
#define AV_FIND_DECODER_CODEC_ID(av_stream)
CriticalSection processingCriticalSection
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
float duration
Length of time (in seconds)
string acodec
The name of the audio codec used to encode / decode the video stream.
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
Scale the clip until both height and width fill the canvas (cropping the overlap) ...
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
#define AVCODEC_REGISTER_ALL
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
int64_t file_size
Size of file (in bytes)
FFmpegReader(string path)
std::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
int audio_bit_rate
The bit rate of the audio stream (in bytes)
bool has_audio
Determines if this file has an audio stream.
This class represents a clip (used to arrange readers on the timeline)
#define AV_FREE_CONTEXT(av_context)
Exception when no valid codec is found for a file.
double Y
The Y value of the coordinate (usually representing the value of the property being animated) ...
int MAX_WIDTH
Maximum width for image data (useful for optimzing for a smaller preview or render) ...
int audio_stream_index
The index of the audio stream.
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
int64_t video_length
The number of frames in the video stream.
#define FF_NUM_PROCESSORS
#define AV_FREE_PACKET(av_packet)
ScaleType scale
The scale determines how a clip should be resized to fit it's parent.
Exception when no streams are found in the file.
int height
The height of the video (in pixels)
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
ClipBase * GetClip()
Parent clip object of this reader (which can be unparented and NULL)
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
auto AV_GET_CODEC_CONTEXT
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Scale the clip until both height and width fill the canvas (distort to fit)
ReaderInfo info
Information about the current media file.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
#define MY_INPUT_BUFFER_PADDING_SIZE
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Exception for frames that are out of bounds.
std::map< string, string > metadata
An optional map/dictionary of metadata for this reader.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
int64_t Count()
Count the frames in the queue.
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Coordinate co
This is the primary coordinate.
Exception for invalid JSON.
#define AV_GET_CODEC_TYPE(av_stream)
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
This struct holds the associated video frame and starting sample # for an audio packet.
Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
void Open()
Open File - which is called by the constructor automatically.
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
int video_bit_rate
The bit rate of the video stream (in bytes)
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
string vcodec
The name of the video codec used to encode / decode the video stream.
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
Scale the clip until either height or width fills the canvas (with no cropping)
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)