28 #include "../include/FrameMapper.h"
31 using namespace openshot;
34 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
73 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.",
"");
76 void FrameMapper::AddField(int64_t frame)
79 AddField(
Field(frame, field_toggle));
82 void FrameMapper::AddField(
Field field)
88 field_toggle = (field_toggle ?
false :
true);
95 void FrameMapper::Init()
97 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Init (Calculate frame mappings)",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
116 if ((fabs(original.
ToFloat() - 24.0) < 1
e-7 || fabs(original.
ToFloat() - 25.0) < 1
e-7 || fabs(original.
ToFloat() - 30.0) < 1
e-7) &&
117 (fabs(target.
ToFloat() - 24.0) < 1
e-7 || fabs(target.
ToFloat() - 25.0) < 1
e-7 || fabs(target.
ToFloat() - 30.0) < 1
e-7)) {
120 float difference = target.
ToInt() - original.
ToInt();
123 int field_interval = 0;
124 int frame_interval = 0;
128 field_interval = round(fabs(original.
ToInt() / difference));
131 frame_interval = field_interval * 2.0f;
140 for (int64_t field = 1; field <= number_of_fields; field++)
148 else if (difference > 0)
158 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
165 AddField(
Field(frame + 1, field_toggle));
167 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
174 else if (difference < 0)
180 field_toggle = (field_toggle ?
false :
true);
182 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
187 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
200 if (field % 2 == 0 && field > 0)
210 double value_increment = (reader->
info.
video_length + 1) / (
double) (new_length);
213 double original_frame_num = 1.0f;
214 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
217 AddField(round(original_frame_num));
218 AddField(round(original_frame_num));
221 original_frame_num += value_increment;
230 int64_t start_samples_frame = 1;
231 int start_samples_position = 0;
233 for (int64_t field = 1; field <=
fields.size(); field++)
239 if (field % 2 == 0 && field > 0)
242 int64_t frame_number = field / 2;
253 int64_t end_samples_frame = start_samples_frame;
254 int end_samples_position = start_samples_position;
257 while (remaining_samples > 0)
263 if (original_samples >= remaining_samples)
266 end_samples_position += remaining_samples - 1;
267 remaining_samples = 0;
271 end_samples_frame += 1;
272 end_samples_position = 0;
273 remaining_samples -= original_samples;
283 start_samples_frame = end_samples_frame;
284 start_samples_position = end_samples_position + 1;
287 start_samples_frame += 1;
288 start_samples_position = 0;
321 frame.
Odd.
Frame = TargetFrameNumber;
331 if(TargetFrameNumber < 1 ||
frames.size() == 0)
335 else if (TargetFrameNumber >
frames.size())
337 TargetFrameNumber =
frames.size();
340 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetMappedFrame",
"TargetFrameNumber", TargetFrameNumber,
"frames.size()",
frames.size(),
"frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
"frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame,
"", -1,
"", -1);
343 return frames[TargetFrameNumber - 1];
347 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
349 std::shared_ptr<Frame> new_frame;
356 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (from reader)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
359 new_frame = reader->
GetFrame(number);
373 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (create blank)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
379 new_frame->AddAudioSilence(samples_in_frame);
387 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
388 if (final_frame)
return final_frame;
399 final_frame = final_cache.
GetFrame(requested_frame);
400 if (final_frame)
return final_frame;
404 int minimum_frames = 1;
407 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (Loop through frames)",
"requested_frame", requested_frame,
"minimum_frames", minimum_frames,
"", -1,
"", -1,
"", -1,
"", -1);
410 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
414 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (inside omp for loop)",
"frame_number", frame_number,
"minimum_frames", minimum_frames,
"requested_frame", requested_frame,
"", -1,
"", -1,
"", -1);
418 std::shared_ptr<Frame> mapped_frame;
421 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
424 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
434 info.
channels == mapped_frame->GetAudioChannelsCount() &&
436 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
439 mapped_frame->number == frame_number &&
443 final_cache.
Add(mapped_frame);
448 std::shared_ptr<Frame> frame = std::make_shared<Frame>(frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
449 frame->SampleRate(mapped_frame->SampleRate());
450 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
454 std::shared_ptr<Frame> odd_frame;
455 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
458 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*odd_frame->GetImage())),
true);
461 std::shared_ptr<Frame> even_frame;
462 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
464 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*even_frame->GetImage())),
false);
468 bool need_resampling =
false;
474 need_resampling =
true;
485 const int EXTRA_INPUT_SAMPLES = 20;
488 copy_samples.
sample_end += EXTRA_INPUT_SAMPLES;
489 int samples_per_end_frame =
492 if (copy_samples.
sample_end >= samples_per_end_frame)
496 copy_samples.
sample_end -= samples_per_end_frame;
498 copy_samples.
total += EXTRA_INPUT_SAMPLES;
505 int samples_per_start_frame =
508 if (copy_samples.
sample_start >= samples_per_start_frame)
514 copy_samples.
total -= EXTRA_INPUT_SAMPLES;
519 int samples_copied = 0;
524 int remaining_samples = copy_samples.
total - samples_copied;
525 int number_to_copy = 0;
528 std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
529 int original_samples = original_frame->GetAudioSamplesCount();
532 for (
int channel = 0; channel < channels_in_frame; channel++)
537 number_to_copy = original_samples - copy_samples.
sample_start;
538 if (number_to_copy > remaining_samples)
539 number_to_copy = remaining_samples;
542 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
547 number_to_copy = original_samples;
548 if (number_to_copy > remaining_samples)
549 number_to_copy = remaining_samples;
552 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
558 if (number_to_copy > remaining_samples)
559 number_to_copy = remaining_samples;
562 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
567 samples_copied += number_to_copy;
577 final_cache.
Add(frame);
582 return final_cache.
GetFrame(requested_frame);
593 float difference = target.
ToInt() - original.
ToInt();
595 int field_interval = 0;
596 int frame_interval = 0;
601 field_interval = round(fabs(original.
ToInt() / difference));
604 frame_interval = field_interval * 2.0f;
608 for (
float map = 1; map <=
frames.size(); map++)
611 cout <<
"Target frame #: " << map <<
" mapped to original frame #:\t(" << frame.
Odd.
Frame <<
" odd, " << frame.
Even.
Frame <<
" even)" << endl;
632 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Open",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
647 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
674 root[
"type"] =
"FrameMapper";
686 bool success = reader.parse( value, root );
689 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
699 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
720 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ChangeMapping",
"target_fps.num", target_fps.
num,
"target_fps.den", target_fps.
den,
"target_pulldown", target_pulldown,
"target_sample_rate", target_sample_rate,
"target_channels", target_channels,
"target_channel_layout", target_channel_layout);
726 target.
num = target_fps.
num;
727 target.
den = target_fps.
den;
732 pulldown = target_pulldown;
760 int total_frame_samples = 0;
761 int channels_in_frame = frame->GetAudioChannelsCount();
762 int sample_rate_in_frame = frame->SampleRate();
763 int samples_in_frame = frame->GetAudioSamplesCount();
764 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
766 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio",
"frame->number", frame->number,
"original_frame_number", original_frame_number,
"channels_in_frame", channels_in_frame,
"samples_in_frame", samples_in_frame,
"sample_rate_in_frame", sample_rate_in_frame,
"", -1);
769 float* frame_samples_float = NULL;
771 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
774 total_frame_samples = samples_in_frame * channels_in_frame;
777 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
780 for (
int s = 0; s < total_frame_samples; s++)
782 frame_samples[s] =
int(frame_samples_float[s] * (1 << 15));
786 delete[] frame_samples_float;
787 frame_samples_float = NULL;
789 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (got sample data from frame)",
"frame->number", frame->number,
"total_frame_samples", total_frame_samples,
"target channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"target sample_rate",
info.
sample_rate,
"samples_in_frame", samples_in_frame);
795 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
797 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
798 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
802 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio ERROR [" + (
string)
av_err2str(error_code) +
"]",
"error_code", error_code,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
803 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
809 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (adjust # of samples)",
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"sample_rate_in_frame", sample_rate_in_frame,
"info.channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"original_frame_number", original_frame_number);
814 audio_converted->nb_samples = total_frame_samples;
815 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
817 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (preparing for resample)",
"in_sample_fmt", AV_SAMPLE_FMT_S16,
"out_sample_fmt", AV_SAMPLE_FMT_S16,
"in_sample_rate", sample_rate_in_frame,
"out_sample_rate",
info.
sample_rate,
"in_channels", channels_in_frame,
"out_channels",
info.
channels);
824 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
826 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
827 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
828 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
830 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
837 audio_converted->data,
838 audio_converted->linesize[0],
839 audio_converted->nb_samples,
841 audio_frame->linesize[0],
842 audio_frame->nb_samples);
845 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
848 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
851 av_freep(&audio_frame->data[0]);
853 av_freep(&audio_converted->data[0]);
855 frame_samples = NULL;
858 int channel_buffer_size = nb_samples;
861 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
"nb_samples", nb_samples,
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"channels_in_frame", channels_in_frame,
"info.channels",
info.
channels,
"info.channel_layout",
info.
channel_layout);
864 float *channel_buffer =
new float[channel_buffer_size];
867 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
870 for (
int z = 0; z < channel_buffer_size; z++)
871 channel_buffer[z] = 0.0f;
877 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
880 if (channel_filter == channel)
883 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
899 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
901 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Add audio to channel)",
"number of samples", position,
"channel_filter", channel_filter,
"", -1,
"", -1,
"", -1,
"", -1);
909 delete[] channel_buffer;
910 channel_buffer = NULL;
913 delete[] resampled_samples;
914 resampled_samples = NULL;
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
ReaderBase * Reader()
Get the current reader.
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
vector< MappedFrame > frames
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
~FrameMapper()
Destructor.
float duration
Length of time (in seconds)
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
This struct holds a single field (half a frame).
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
Exception when encoding audio packet.
This struct holds a the range of samples needed by this frame.
virtual std::shared_ptr< Frame > GetFrame(int64_t number)=0
bool has_audio
Determines if this file has an audio stream.
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
int height
The height of the video (in pixels)
#define AV_ALLOCATE_FRAME()
bool IsOpen()
Determine if reader is open or closed.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
bool has_single_image
Determines if this file only contains a single image.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
#define av_err2str(errnum)
ReaderInfo info
Information about the current media file.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
This struct holds two fields which together make up a complete video frame.
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
string Json()
Get and Set JSON methods.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Do not apply pull-down techniques, just repeat or skip entire frames.
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
PulldownType
This enumeration determines how frame rates are increased or decreased.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
void SetJson(string value)
Load JSON string into this object.
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.
virtual bool IsOpen()=0
Determine if reader is open or closed.