28 #include "../include/FrameMapper.h"
31 using namespace openshot;
34 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
76 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.",
"");
79 void FrameMapper::AddField(int64_t frame)
82 AddField(
Field(frame, field_toggle));
85 void FrameMapper::AddField(
Field field)
91 field_toggle = (field_toggle ?
false :
true);
98 void FrameMapper::Init()
100 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Init (Calculate frame mappings)",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
119 if ((fabs(original.
ToFloat() - 24.0) < 1
e-7 || fabs(original.
ToFloat() - 25.0) < 1
e-7 || fabs(original.
ToFloat() - 30.0) < 1
e-7) &&
120 (fabs(target.
ToFloat() - 24.0) < 1
e-7 || fabs(target.
ToFloat() - 25.0) < 1
e-7 || fabs(target.
ToFloat() - 30.0) < 1
e-7)) {
123 float difference = target.
ToInt() - original.
ToInt();
126 int field_interval = 0;
127 int frame_interval = 0;
131 field_interval = round(fabs(original.
ToInt() / difference));
134 frame_interval = field_interval * 2.0f;
143 for (int64_t field = 1; field <= number_of_fields; field++)
151 else if (difference > 0)
161 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
168 AddField(
Field(frame + 1, field_toggle));
170 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
177 else if (difference < 0)
183 field_toggle = (field_toggle ?
false :
true);
185 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
190 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
203 if (field % 2 == 0 && field > 0)
219 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
222 AddField(rate_curve.
GetInt(frame_num));
223 AddField(rate_curve.
GetInt(frame_num));
232 int64_t start_samples_frame = 1;
233 int start_samples_position = 0;
235 for (int64_t field = 1; field <=
fields.size(); field++)
241 if (field % 2 == 0 && field > 0)
244 int64_t frame_number = field / 2;
255 int64_t end_samples_frame = start_samples_frame;
256 int end_samples_position = start_samples_position;
259 while (remaining_samples > 0)
265 if (original_samples >= remaining_samples)
268 end_samples_position += remaining_samples - 1;
269 remaining_samples = 0;
273 end_samples_frame += 1;
274 end_samples_position = 0;
275 remaining_samples -= original_samples;
285 start_samples_frame = end_samples_frame;
286 start_samples_position = end_samples_position + 1;
289 start_samples_frame += 1;
290 start_samples_position = 0;
318 frame.
Odd.
Frame = TargetFrameNumber;
328 if(TargetFrameNumber < 1 ||
frames.size() == 0)
332 else if (TargetFrameNumber >
frames.size())
334 TargetFrameNumber =
frames.size();
337 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetMappedFrame",
"TargetFrameNumber", TargetFrameNumber,
"frames.size()",
frames.size(),
"frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
"frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame,
"", -1,
"", -1);
340 return frames[TargetFrameNumber - 1];
344 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
346 std::shared_ptr<Frame> new_frame;
353 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (from reader)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
359 new_frame = reader->
GetFrame(number);
373 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (create blank)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
379 new_frame->AddAudioSilence(samples_in_frame);
387 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
388 if (final_frame)
return final_frame;
399 final_frame = final_cache.
GetFrame(requested_frame);
400 if (final_frame)
return final_frame;
404 int minimum_frames = 1;
407 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (Loop through frames)",
"requested_frame", requested_frame,
"minimum_frames", minimum_frames,
"", -1,
"", -1,
"", -1,
"", -1);
410 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
414 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (inside omp for loop)",
"frame_number", frame_number,
"minimum_frames", minimum_frames,
"requested_frame", requested_frame,
"", -1,
"", -1,
"", -1);
418 std::shared_ptr<Frame> mapped_frame;
421 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
424 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
434 info.
channels == mapped_frame->GetAudioChannelsCount() &&
436 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
439 mapped_frame->number == frame_number &&
443 final_cache.
Add(mapped_frame);
448 std::shared_ptr<Frame> frame = std::make_shared<Frame>(frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
449 frame->SampleRate(mapped_frame->SampleRate());
450 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
454 std::shared_ptr<Frame> odd_frame;
455 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
458 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*odd_frame->GetImage())),
true);
461 std::shared_ptr<Frame> even_frame;
462 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
464 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*even_frame->GetImage())),
false);
468 bool need_resampling =
false;
474 need_resampling =
true;
485 const int EXTRA_INPUT_SAMPLES = 20;
488 copy_samples.
sample_end += EXTRA_INPUT_SAMPLES;
489 int samples_per_end_frame =
492 if (copy_samples.
sample_end >= samples_per_end_frame)
496 copy_samples.
sample_end -= samples_per_end_frame;
498 copy_samples.
total += EXTRA_INPUT_SAMPLES;
505 int samples_per_start_frame =
508 if (copy_samples.
sample_start >= samples_per_start_frame)
514 copy_samples.
total -= EXTRA_INPUT_SAMPLES;
519 int samples_copied = 0;
524 int remaining_samples = copy_samples.
total - samples_copied;
525 int number_to_copy = 0;
528 std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
529 int original_samples = original_frame->GetAudioSamplesCount();
532 for (
int channel = 0; channel < channels_in_frame; channel++)
537 number_to_copy = original_samples - copy_samples.
sample_start;
538 if (number_to_copy > remaining_samples)
539 number_to_copy = remaining_samples;
542 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
547 number_to_copy = original_samples;
548 if (number_to_copy > remaining_samples)
549 number_to_copy = remaining_samples;
552 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
558 if (number_to_copy > remaining_samples)
559 number_to_copy = remaining_samples;
562 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
567 samples_copied += number_to_copy;
577 final_cache.
Add(frame);
582 return final_cache.
GetFrame(requested_frame);
593 float difference = target.
ToInt() - original.
ToInt();
595 int field_interval = 0;
596 int frame_interval = 0;
601 field_interval = round(fabs(original.
ToInt() / difference));
604 frame_interval = field_interval * 2.0f;
608 for (
float map = 1; map <=
frames.size(); map++)
611 cout <<
"Target frame #: " << map <<
" mapped to original frame #:\t(" << frame.
Odd.
Frame <<
" odd, " << frame.
Even.
Frame <<
" even)" << endl;
632 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Open",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
647 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
674 root[
"type"] =
"FrameMapper";
686 bool success = reader.parse( value, root );
689 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
699 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
720 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ChangeMapping",
"target_fps.num", target_fps.
num,
"target_fps.den", target_fps.
den,
"target_pulldown", target_pulldown,
"target_sample_rate", target_sample_rate,
"target_channels", target_channels,
"target_channel_layout", target_channel_layout);
726 target.
num = target_fps.
num;
727 target.
den = target_fps.
den;
732 pulldown = target_pulldown;
758 int total_frame_samples = 0;
759 int channels_in_frame = frame->GetAudioChannelsCount();
760 int sample_rate_in_frame = frame->SampleRate();
761 int samples_in_frame = frame->GetAudioSamplesCount();
762 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
764 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio",
"frame->number", frame->number,
"original_frame_number", original_frame_number,
"channels_in_frame", channels_in_frame,
"samples_in_frame", samples_in_frame,
"sample_rate_in_frame", sample_rate_in_frame,
"", -1);
767 float* frame_samples_float = NULL;
769 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
772 total_frame_samples = samples_in_frame * channels_in_frame;
775 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
778 for (
int s = 0; s < total_frame_samples; s++)
780 frame_samples[s] =
int(frame_samples_float[s] * (1 << 15));
784 delete[] frame_samples_float;
785 frame_samples_float = NULL;
787 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (got sample data from frame)",
"frame->number", frame->number,
"total_frame_samples", total_frame_samples,
"target channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"target sample_rate",
info.
sample_rate,
"samples_in_frame", samples_in_frame);
793 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
795 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
796 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
800 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio ERROR [" + (
string)
av_err2str(error_code) +
"]",
"error_code", error_code,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
801 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
807 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (adjust # of samples)",
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"sample_rate_in_frame", sample_rate_in_frame,
"info.channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"original_frame_number", original_frame_number);
812 audio_converted->nb_samples = total_frame_samples;
813 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
815 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (preparing for resample)",
"in_sample_fmt", AV_SAMPLE_FMT_S16,
"out_sample_fmt", AV_SAMPLE_FMT_S16,
"in_sample_rate", sample_rate_in_frame,
"out_sample_rate",
info.
sample_rate,
"in_channels", channels_in_frame,
"out_channels",
info.
channels);
822 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
824 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
825 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
826 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
828 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
835 audio_converted->data,
836 audio_converted->linesize[0],
837 audio_converted->nb_samples,
839 audio_frame->linesize[0],
840 audio_frame->nb_samples);
843 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
846 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
849 av_freep(&audio_frame->data[0]);
851 av_freep(&audio_converted->data[0]);
853 frame_samples = NULL;
856 int channel_buffer_size = nb_samples;
859 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
"nb_samples", nb_samples,
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"channels_in_frame", channels_in_frame,
"info.channels",
info.
channels,
"info.channel_layout",
info.
channel_layout);
862 float *channel_buffer =
new float[channel_buffer_size];
865 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
868 for (
int z = 0; z < channel_buffer_size; z++)
869 channel_buffer[z] = 0.0f;
875 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
878 if (channel_filter == channel)
881 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
897 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
899 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Add audio to channel)",
"number of samples", position,
"channel_filter", channel_filter,
"", -1,
"", -1,
"", -1,
"", -1);
907 delete[] channel_buffer;
908 channel_buffer = NULL;
911 delete[] resampled_samples;
912 resampled_samples = NULL;
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
int max_height
The maximium image height needed by this clip (used for optimizations)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
ReaderBase * Reader()
Get the current reader.
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
vector< MappedFrame > frames
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
~FrameMapper()
Destructor.
float duration
Length of time (in seconds)
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
This struct holds a single field (half a frame).
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
Exception when encoding audio packet.
void AddPoint(Point p)
Add a new point on the key-frame. Each point has a primary coordinate, a left handle, and a right handle.
This struct holds a the range of samples needed by this frame.
virtual std::shared_ptr< Frame > GetFrame(int64_t number)=0
void SetMaxSize(int width, int height)
Set Max Image Size (used for performance optimization)
bool has_audio
Determines if this file has an audio stream.
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
int height
The height of the video (in pixels)
#define AV_ALLOCATE_FRAME()
bool IsOpen()
Determine if reader is open or closed.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
bool has_single_image
Determines if this file only contains a single image.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
int GetInt(int64_t index)
Get the rounded INT value at a specific index.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
#define av_err2str(errnum)
ReaderInfo info
Information about the current media file.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
This struct holds two fields which together make up a complete video frame.
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
string Json()
Get and Set JSON methods.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Do not apply pull-down techniques, just repeat or skip entire frames.
Linear curves are angular, straight lines between two points.
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
PulldownType
This enumeration determines how frame rates are increased or decreased.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
A Keyframe is a collection of Point instances, which is used to vary a number or property over time...
int max_width
The maximum image width needed by this clip (used for optimizations)
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
void SetJson(string value)
Load JSON string into this object.
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.
virtual bool IsOpen()=0
Determine if reader is open or closed.