28 #include "../include/Frame.h"
31 using namespace openshot;
34 Frame::Frame() : number(1), pixel_ratio(1,1), channels(2), width(1), height(1), color(
"#000000"),
35 channel_layout(
LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
39 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, 0));
47 : number(number), pixel_ratio(1,1), channels(2), width(width), height(height), color(color),
48 channel_layout(
LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
52 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, 0));
60 number(number), pixel_ratio(1,1), channels(channels), width(1), height(1), color(
"#000000"),
61 channel_layout(
LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
65 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, samples));
72 Frame::Frame(int64_t number,
int width,
int height,
string color,
int samples,
int channels)
73 : number(number), pixel_ratio(1,1), channels(channels), width(width), height(height), color(color),
74 channel_layout(
LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
78 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, samples));
105 channels = other.channels;
107 height = other.height;
108 channel_layout = other.channel_layout;
111 sample_rate = other.sample_rate;
112 pixel_ratio =
Fraction(other.pixel_ratio.
num, other.pixel_ratio.
den);
116 image = std::shared_ptr<QImage>(
new QImage(*(other.image)));
118 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(*(other.audio)));
119 if (other.wave_image)
120 wave_image = std::shared_ptr<QImage>(
new QImage(*(other.wave_image)));
133 if (!QApplication::instance()) {
136 static char* argv[1] = {NULL};
137 previewApp = std::shared_ptr<QApplication>(
new QApplication(argc, argv));
141 std::shared_ptr<QImage> previewImage =
GetImage();
144 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
147 int new_width = previewImage->size().width();
151 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
155 QWidget previewWindow;
156 previewWindow.setStyleSheet(
"background-color: #000000;");
161 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
162 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
163 layout.addWidget(&previewLabel);
166 previewWindow.setLayout(&layout);
167 previewWindow.show();
172 std::shared_ptr<QImage>
Frame::GetWaveform(
int width,
int height,
int Red,
int Green,
int Blue,
int Alpha)
178 QVector<QPointF> lines;
179 QVector<QPointF> labels;
183 if (total_samples > 0)
186 int new_height = 200 * audio->getNumChannels();
187 int height_padding = 20 * (audio->getNumChannels() - 1);
188 int total_height = new_height + height_padding;
193 for (
int channel = 0; channel < audio->getNumChannels(); channel++)
198 const float *samples = audio->getReadPointer(channel);
203 float value = samples[sample] * 100;
208 lines.push_back(QPointF(X,Y));
209 lines.push_back(QPointF(X,Y-value));
213 lines.push_back(QPointF(X,Y));
214 lines.push_back(QPointF(X,Y));
219 labels.push_back(QPointF(5, Y - 5));
222 Y += (200 + height_padding);
227 wave_image = std::shared_ptr<QImage>(
new QImage(total_width, total_height, QImage::Format_RGBA8888));
228 wave_image->fill(QColor(0,0,0,0));
231 QPainter painter(wave_image.get());
234 painter.setPen(QColor(Red, Green, Blue, Alpha));
237 painter.drawLines(lines);
250 if (width != total_width || height != total_height) {
251 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::FastTransformation);
252 wave_image = std::shared_ptr<QImage>(
new QImage(scaled_wave_image));
258 wave_image = std::shared_ptr<QImage>(
new QImage(width, height, QImage::Format_RGBA8888));
259 wave_image->fill(QColor(QString::fromStdString(
"#000000")));
277 wave_image =
GetWaveform(width, height, Red, Green, Blue, Alpha);
280 return wave_image->bits();
289 if (!QApplication::instance()) {
292 static char* argv[1] = {NULL};
293 previewApp = std::shared_ptr<QApplication>(
new QApplication(argc, argv));
297 QWidget previewWindow;
298 previewWindow.setStyleSheet(
"background-color: #000000;");
303 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
304 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
305 layout.addWidget(&previewLabel);
308 previewWindow.setLayout(&layout);
309 previewWindow.show();
321 return audio->getMagnitude(channel, sample, magnitude_range);
325 return audio->getMagnitude(sample, magnitude_range);
333 return audio->getWritePointer(channel);
339 float *output = NULL;
340 AudioSampleBuffer *buffer(audio.get());
341 int num_of_channels = audio->getNumChannels();
345 if (new_sample_rate != sample_rate)
348 resampler->
SetBuffer(audio.get(), sample_rate, new_sample_rate);
354 num_of_samples = buffer->getNumSamples();
358 output =
new float[num_of_channels * num_of_samples];
362 for (
int channel = 0; channel < num_of_channels; channel++)
364 for (
int sample = 0; sample < num_of_samples; sample++)
367 output[position] = buffer->getReadPointer(channel)[sample];
375 *sample_count = num_of_samples;
385 float *output = NULL;
386 AudioSampleBuffer *buffer(audio.get());
387 int num_of_channels = audio->getNumChannels();
391 if (new_sample_rate != sample_rate && resampler)
394 resampler->
SetBuffer(audio.get(), sample_rate, new_sample_rate);
400 num_of_samples = buffer->getNumSamples();
404 output =
new float[num_of_channels * num_of_samples];
408 for (
int sample = 0; sample < num_of_samples; sample++)
410 for (
int channel = 0; channel < num_of_channels; channel++)
413 output[position] = buffer->getReadPointer(channel)[sample];
421 *sample_count = num_of_samples;
430 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
432 return audio->getNumChannels();
440 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
441 return max_audio_sample;
452 int64_t total_bytes = 0;
454 total_bytes += (width * height *
sizeof(char) * 4);
457 total_bytes += (sample_rate / 24.0) *
sizeof(float);
473 return image->bits();
480 return image->scanLine(row);
486 pixel_ratio.
num = num;
487 pixel_ratio.
den = den;
503 double previous_samples = (sample_rate * fps_rate) * (number - 1);
504 double previous_samples_remainder = fmod(previous_samples, (
double)channels);
505 previous_samples -= previous_samples_remainder;
508 double total_samples = (sample_rate * fps_rate) * number;
509 double total_samples_remainder = fmod(total_samples, (
double)channels);
510 total_samples -= total_samples_remainder;
514 int samples_per_frame = round(total_samples - previous_samples);
515 if (samples_per_frame < 0)
516 samples_per_frame = 0;
517 return samples_per_frame;
547 return channel_layout;
552 void Frame::Save(
string path,
float scale,
string format,
int quality)
555 std::shared_ptr<QImage> previewImage =
GetImage();
558 if (abs(scale) > 1.001 || abs(scale) < 0.999)
560 int new_width = width;
561 int new_height = height;
564 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
567 int new_width = previewImage->size().width();
571 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
575 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width * scale, new_height * scale, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
579 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
583 void Frame::Thumbnail(
string path,
int new_width,
int new_height,
string mask_path,
string overlay_path,
584 string background_color,
bool ignore_aspect,
string format,
int quality,
float rotate) {
587 std::shared_ptr<QImage> thumbnail = std::shared_ptr<QImage>(
new QImage(new_width, new_height, QImage::Format_RGBA8888));
588 thumbnail->fill(QColor(QString::fromStdString(background_color)));
591 QPainter painter(thumbnail.get());
592 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing,
true);
595 std::shared_ptr<QImage> previewImage =
GetImage();
598 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
601 int aspect_width = previewImage->size().width();
602 int aspect_height = previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble();
605 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(aspect_width, aspect_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
611 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
614 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
617 int x = (new_width - previewImage->size().width()) / 2.0;
618 int y = (new_height - previewImage->size().height()) / 2.0;
619 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
623 QTransform transform;
624 float origin_x = previewImage->width() / 2.0;
625 float origin_y = previewImage->height() / 2.0;
626 transform.translate(origin_x, origin_y);
627 transform.rotate(rotate);
628 transform.translate(-origin_x,-origin_y);
629 painter.setTransform(transform);
632 painter.drawImage(x, y, *previewImage);
636 if (overlay_path !=
"") {
638 std::shared_ptr<QImage> overlay = std::shared_ptr<QImage>(
new QImage());
639 overlay->load(QString::fromStdString(overlay_path));
642 overlay = std::shared_ptr<QImage>(
new QImage(overlay->convertToFormat(QImage::Format_RGBA8888)));
645 overlay = std::shared_ptr<QImage>(
new QImage(overlay->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
648 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
649 painter.drawImage(0, 0, *overlay);
654 if (mask_path !=
"") {
656 std::shared_ptr<QImage> mask = std::shared_ptr<QImage>(
new QImage());
657 mask->load(QString::fromStdString(mask_path));
660 mask = std::shared_ptr<QImage>(
new QImage(mask->convertToFormat(QImage::Format_RGBA8888)));
663 mask = std::shared_ptr<QImage>(
new QImage(mask->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
666 mask->invertPixels();
669 unsigned char *pixels = (
unsigned char *) thumbnail->bits();
670 unsigned char *mask_pixels = (
unsigned char *) mask->bits();
674 for (
int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
677 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
678 int Frame_Alpha = pixels[byte_index + 3];
679 int Mask_Value = constrain(Frame_Alpha - gray_value);
682 pixels[byte_index + 3] = Mask_Value;
691 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
695 int Frame::constrain(
int color_value)
700 else if (color_value > 255)
713 const GenericScopedLock<CriticalSection> lock(addingImageSection);
714 #pragma omp critical (AddImage)
716 image = std::shared_ptr<QImage>(
new QImage(new_width, new_height, QImage::Format_RGBA8888));
719 image->fill(QColor(QString::fromStdString(color)));
722 width = image->width();
723 height = image->height();
728 void Frame::AddImage(
int new_width,
int new_height,
int bytes_per_pixel, QImage::Format type,
const unsigned char *pixels_)
731 const GenericScopedLock<CriticalSection> lock(addingImageSection);
732 int buffer_size = new_width * new_height * bytes_per_pixel;
733 qbuffer =
new unsigned char[buffer_size]();
736 memcpy((
unsigned char*)qbuffer, pixels_, buffer_size);
739 #pragma omp critical (AddImage)
741 image = std::shared_ptr<QImage>(
new QImage(qbuffer, new_width, new_height, new_width * bytes_per_pixel, type, (QImageCleanupFunction) &
openshot::Frame::cleanUpBuffer, (
void*) qbuffer));
744 if (image->format() != QImage::Format_RGBA8888)
745 *image = image->convertToFormat(QImage::Format_RGBA8888);
748 width = image->width();
749 height = image->height();
762 const GenericScopedLock<CriticalSection> lock(addingImageSection);
763 #pragma omp critical (AddImage)
768 if (image->format() != QImage::Format_RGBA8888)
769 *image = image->convertToFormat(QImage::Format_RGBA8888);
772 width = image->width();
773 height = image->height();
794 #pragma omp critical (AddImage)
795 if (image == new_image || image->size() != image->size() || image->format() != image->format())
801 const GenericScopedLock<CriticalSection> lock(addingImageSection);
802 #pragma omp critical (AddImage)
804 const unsigned char *pixels = image->bits();
805 const unsigned char *new_pixels = new_image->bits();
811 for (
int row = start; row < image->height(); row += 2) {
812 memcpy((
unsigned char *) pixels, new_pixels + (row * image->bytesPerLine()), image->bytesPerLine());
813 new_pixels += image->bytesPerLine();
817 width = image->width();
818 height = image->height();
828 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
831 audio->setSize(channels, length,
true,
true,
false);
832 channel_layout = layout;
836 max_audio_sample = length;
840 void Frame::AddAudio(
bool replaceSamples,
int destChannel,
int destStartSample,
const float* source,
int numSamples,
float gainToApplyToSource = 1.0f) {
841 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
842 #pragma omp critical (adding_audio)
845 int destStartSampleAdjusted = max(destStartSample, 0);
848 int new_length = destStartSampleAdjusted + numSamples;
849 int new_channel_length = audio->getNumChannels();
850 if (destChannel >= new_channel_length)
851 new_channel_length = destChannel + 1;
852 if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
853 audio->setSize(new_channel_length, new_length,
true,
true,
false);
857 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
860 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
864 if (new_length > max_audio_sample)
865 max_audio_sample = new_length;
870 void Frame::ApplyGainRamp(
int destChannel,
int destStartSample,
int numSamples,
float initial_gain = 0.0f,
float final_gain = 1.0f)
872 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
875 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
889 #ifdef USE_IMAGEMAGICK
891 std::shared_ptr<Magick::Image> Frame::GetMagickImage()
899 QRgb
const *tmpBits = (
const QRgb*)image->bits();
902 std::shared_ptr<Magick::Image> magick_image = std::shared_ptr<Magick::Image>(
new Magick::Image(image->width(), image->height(),
"RGBA", Magick::CharPixel, tmpBits));
905 magick_image->backgroundColor(Magick::Color(
"none"));
906 magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
907 magick_image->matte(
true);
913 #ifdef USE_IMAGEMAGICK
915 void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
918 const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
923 qbuffer =
new unsigned char[bufferSize]();
924 unsigned char *buffer = (
unsigned char*)qbuffer;
929 Magick::PixelPacket *pixels = new_image->getPixels(0,0, new_image->columns(), new_image->rows());
930 for (
int n = 0, i = 0; n < new_image->columns() * new_image->rows(); n += 1, i += 4) {
931 buffer[i+0] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].red);
932 buffer[i+1] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].green);
933 buffer[i+2] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].blue);
934 buffer[i+3] = 255 - MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].opacity);
939 image = std::shared_ptr<QImage>(
new QImage(qbuffer, width, height, width * BPP, QImage::Format_RGBA8888, (QImageCleanupFunction) &
cleanUpBuffer, (
void*) qbuffer));
942 width = image->width();
943 height = image->height();
955 AudioDeviceManager deviceManager;
956 deviceManager.initialise (0,
962 AudioSourcePlayer audioSourcePlayer;
963 deviceManager.addAudioCallback (&audioSourcePlayer);
965 ScopedPointer<AudioBufferSource> my_source;
969 TimeSliceThread my_thread(
"Audio buffer thread");
972 my_thread.startThread();
974 AudioTransportSource transport1;
975 transport1.setSource (my_source,
978 (
double) sample_rate,
979 audio->getNumChannels());
980 transport1.setPosition (0);
981 transport1.setGain(1.0);
985 MixerAudioSource mixer;
986 mixer.addInputSource(&transport1,
false);
987 audioSourcePlayer.setSource (&mixer);
992 while (transport1.isPlaying())
994 cout <<
"playing" << endl;
998 cout <<
"DONE!!!" << endl;
1001 transport1.setSource (0);
1002 audioSourcePlayer.setSource (0);
1003 my_thread.stopThread(500);
1004 deviceManager.removeAudioCallback (&audioSourcePlayer);
1005 deviceManager.closeAudioDevice();
1006 deviceManager.removeAllChangeListeners();
1007 deviceManager.dispatchPendingMessages();
1009 cout <<
"End of Play()" << endl;
1020 unsigned char* ptr_to_qbuffer = (
unsigned char*) info;
1021 delete[] ptr_to_qbuffer;
1028 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
1031 audio->setSize(channels, numSamples,
false,
true,
false);
1036 if (numSamples > max_audio_sample)
1037 max_audio_sample = numSamples;
void SetBuffer(AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
int GetWidth()
Get height of image.
int num
Numerator for the fraction.
int GetAudioSamplesCount()
Get number of audio samples.
float * GetInterleavedAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
This class represents a single frame of video (i.e. image & audio data)
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
juce::AudioSampleBuffer * GetAudioSampleBuffer()
const unsigned char * GetPixels()
Get pixel data (as packets)
void Play()
Play audio samples for this frame.
void Save(string path, float scale, string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Fraction Reciprocal()
Return the reciprocal as a Fraction.
int64_t number
This is the frame number (starting at 1)
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
Frame & operator=(const Frame &other)
Assignment operator.
void ClearWaveform()
Clear the waveform image (and deallocate it's memory)
void AddColor(int new_width, int new_height, string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
float * GetAudioSamples(int channel)
Get an array of sample data.
void SetFrameNumber(int64_t number)
Set frame number.
void AddAudioSilence(int numSamples)
Add audio silence.
bool has_audio_data
This frame has been loaded with audio data.
This class represents a fraction.
void Thumbnail(string path, int new_width, int new_height, string mask_path, string overlay_path, string background_color, bool ignore_aspect, string format="png", int quality=100, float rotate=0.0)
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
void DisplayWaveform()
Display the wave form.
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
int GetAudioChannelsCount()
Get number of audio channels.
ChannelLayout ChannelsLayout()
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
float * GetPlanarAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
int den
Denominator for the fraction.
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
bool has_image_data
This frame has been loaded with pixel data.
void ResizeAudio(int channels, int length, int sample_rate, ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
int GetHeight()
Get height of image.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int SampleRate()
Get the original sample rate of this frame's audio data.
This class is used to resample audio data for many sequential frames.