OpenShot Library | libopenshot  0.2.3
Frame.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for Frame class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2014 OpenShot Studios, LLC
9  * <http://www.openshotstudios.com/>. This file is part of
10  * OpenShot Library (libopenshot), an open-source project dedicated to
11  * delivering high quality video editing and animation solutions to the
12  * world. For more information visit <http://www.openshot.org/>.
13  *
14  * OpenShot Library (libopenshot) is free software: you can redistribute it
15  * and/or modify it under the terms of the GNU Lesser General Public License
16  * as published by the Free Software Foundation, either version 3 of the
17  * License, or (at your option) any later version.
18  *
19  * OpenShot Library (libopenshot) is distributed in the hope that it will be
20  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22  * GNU Lesser General Public License for more details.
23  *
24  * You should have received a copy of the GNU Lesser General Public License
25  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
26  */
27 
28 #include "../include/Frame.h"
29 
30 using namespace std;
31 using namespace openshot;
32 
33 // Constructor - blank frame (300x200 blank image, 48kHz audio silence)
34 Frame::Frame() : number(1), pixel_ratio(1,1), channels(2), width(1), height(1), color("#000000"),
35  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
36  max_audio_sample(0)
37 {
38  // Init the image magic and audio buffer
39  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, 0));
40 
41  // initialize the audio samples to zero (silence)
42  audio->clear();
43 };
44 
45 // Constructor - image only (48kHz audio silence)
46 Frame::Frame(int64_t number, int width, int height, string color)
47  : number(number), pixel_ratio(1,1), channels(2), width(width), height(height), color(color),
48  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
49  max_audio_sample(0)
50 {
51  // Init the image magic and audio buffer
52  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, 0));
53 
54  // initialize the audio samples to zero (silence)
55  audio->clear();
56 };
57 
58 // Constructor - audio only (300x200 blank image)
59 Frame::Frame(int64_t number, int samples, int channels) :
60  number(number), pixel_ratio(1,1), channels(channels), width(1), height(1), color("#000000"),
61  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
62  max_audio_sample(0)
63 {
64  // Init the image magic and audio buffer
65  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, samples));
66 
67  // initialize the audio samples to zero (silence)
68  audio->clear();
69 };
70 
71 // Constructor - image & audio
72 Frame::Frame(int64_t number, int width, int height, string color, int samples, int channels)
73  : number(number), pixel_ratio(1,1), channels(channels), width(width), height(height), color(color),
74  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
75  max_audio_sample(0)
76 {
77  // Init the image magic and audio buffer
78  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, samples));
79 
80  // initialize the audio samples to zero (silence)
81  audio->clear();
82 };
83 
84 
85 // Copy constructor
86 Frame::Frame ( const Frame &other )
87 {
88  // copy pointers and data
89  DeepCopy(other);
90 }
91 
92 // Assignment operator
94 {
95  // copy pointers and data
96  DeepCopy(other);
97 
98  return *this;
99 }
100 
101 // Copy data and pointers from another Frame instance
102 void Frame::DeepCopy(const Frame& other)
103 {
104  number = other.number;
105  channels = other.channels;
106  width = other.width;
107  height = other.height;
108  channel_layout = other.channel_layout;
111  sample_rate = other.sample_rate;
112  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
113  color = other.color;
114 
115  if (other.image)
116  image = std::shared_ptr<QImage>(new QImage(*(other.image)));
117  if (other.audio)
118  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(*(other.audio)));
119  if (other.wave_image)
120  wave_image = std::shared_ptr<QImage>(new QImage(*(other.wave_image)));
121 }
122 
123 // Descructor
125  // Clear all pointers
126  image.reset();
127  audio.reset();
128 }
129 
130 // Display the frame image to the screen (primarily used for debugging reasons)
132 {
133  if (!QApplication::instance()) {
134  // Only create the QApplication once
135  static int argc = 1;
136  static char* argv[1] = {NULL};
137  previewApp = std::shared_ptr<QApplication>(new QApplication(argc, argv));
138  }
139 
140  // Get preview image
141  std::shared_ptr<QImage> previewImage = GetImage();
142 
143  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
144  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
145  {
146  // Calculate correct DAR (display aspect ratio)
147  int new_width = previewImage->size().width();
148  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
149 
150  // Resize to fix DAR
151  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
152  }
153 
154  // Create window
155  QWidget previewWindow;
156  previewWindow.setStyleSheet("background-color: #000000;");
157  QHBoxLayout layout;
158 
159  // Create label with current frame's image
160  QLabel previewLabel;
161  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
162  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
163  layout.addWidget(&previewLabel);
164 
165  // Show the window
166  previewWindow.setLayout(&layout);
167  previewWindow.show();
168  previewApp->exec();
169 }
170 
171 // Get an audio waveform image
172 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
173 {
174  // Clear any existing waveform image
175  ClearWaveform();
176 
177  // Init a list of lines
178  QVector<QPointF> lines;
179  QVector<QPointF> labels;
180 
181  // Calculate width of an image based on the # of samples
182  int total_samples = GetAudioSamplesCount();
183  if (total_samples > 0)
184  {
185  // If samples are present...
186  int new_height = 200 * audio->getNumChannels();
187  int height_padding = 20 * (audio->getNumChannels() - 1);
188  int total_height = new_height + height_padding;
189  int total_width = 0;
190 
191  // Loop through each audio channel
192  int Y = 100;
193  for (int channel = 0; channel < audio->getNumChannels(); channel++)
194  {
195  int X = 0;
196 
197  // Get audio for this channel
198  const float *samples = audio->getReadPointer(channel);
199 
200  for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
201  {
202  // Sample value (scaled to -100 to 100)
203  float value = samples[sample] * 100;
204 
205  // Append a line segment for each sample
206  if (value != 0.0) {
207  // LINE
208  lines.push_back(QPointF(X,Y));
209  lines.push_back(QPointF(X,Y-value));
210  }
211  else {
212  // DOT
213  lines.push_back(QPointF(X,Y));
214  lines.push_back(QPointF(X,Y));
215  }
216  }
217 
218  // Add Channel Label Coordinate
219  labels.push_back(QPointF(5, Y - 5));
220 
221  // Increment Y
222  Y += (200 + height_padding);
223  total_width = X;
224  }
225 
226  // Create blank image
227  wave_image = std::shared_ptr<QImage>(new QImage(total_width, total_height, QImage::Format_RGBA8888));
228  wave_image->fill(QColor(0,0,0,0));
229 
230  // Load QPainter with wave_image device
231  QPainter painter(wave_image.get());
232 
233  // Set pen color
234  painter.setPen(QColor(Red, Green, Blue, Alpha));
235 
236  // Draw the waveform
237  painter.drawLines(lines);
238  painter.end();
239 
240  // Loop through the channels labels (and draw the text)
241  // TODO: Configure Fonts in Qt5 correctly, so the drawText method does not crash
242 // painter.setFont(QFont(QString("Arial"), 16, 1, false));
243 // for (int channel = 0; channel < labels.size(); channel++) {
244 // stringstream label;
245 // label << "Channel " << channel;
246 // painter.drawText(labels.at(channel), QString::fromStdString(label.str()));
247 // }
248 
249  // Resize Image (if requested)
250  if (width != total_width || height != total_height) {
251  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::FastTransformation);
252  wave_image = std::shared_ptr<QImage>(new QImage(scaled_wave_image));
253  }
254  }
255  else
256  {
257  // No audio samples present
258  wave_image = std::shared_ptr<QImage>(new QImage(width, height, QImage::Format_RGBA8888));
259  wave_image->fill(QColor(QString::fromStdString("#000000")));
260  }
261 
262  // Return new image
263  return wave_image;
264 }
265 
266 // Clear the waveform image (and deallocate it's memory)
268 {
269  if (wave_image)
270  wave_image.reset();
271 }
272 
273 // Get an audio waveform image pixels
274 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
275 {
276  // Get audio wave form image
277  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
278 
279  // Return array of pixel packets
280  return wave_image->bits();
281 }
282 
283 // Display the wave form
285 {
286  // Get audio wave form image
287  GetWaveform(720, 480, 0, 123, 255, 255);
288 
289  if (!QApplication::instance()) {
290  // Only create the QApplication once
291  static int argc = 1;
292  static char* argv[1] = {NULL};
293  previewApp = std::shared_ptr<QApplication>(new QApplication(argc, argv));
294  }
295 
296  // Create window
297  QWidget previewWindow;
298  previewWindow.setStyleSheet("background-color: #000000;");
299  QHBoxLayout layout;
300 
301  // Create label with current frame's waveform image
302  QLabel previewLabel;
303  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
304  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
305  layout.addWidget(&previewLabel);
306 
307  // Show the window
308  previewWindow.setLayout(&layout);
309  previewWindow.show();
310  previewApp->exec();
311 
312  // Deallocate waveform image
313  ClearWaveform();
314 }
315 
316 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
317 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
318 {
319  if (channel > 0) {
320  // return average magnitude for a specific channel/sample range
321  return audio->getMagnitude(channel, sample, magnitude_range);
322 
323  } else {
324  // Return average magnitude for all channels
325  return audio->getMagnitude(sample, magnitude_range);
326  }
327 }
328 
329 // Get an array of sample data
330 float* Frame::GetAudioSamples(int channel)
331 {
332  // return JUCE audio data for this channel
333  return audio->getWritePointer(channel);
334 }
335 
336 // Get a planar array of sample data, using any sample rate
337 float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
338 {
339  float *output = NULL;
340  AudioSampleBuffer *buffer(audio.get());
341  int num_of_channels = audio->getNumChannels();
342  int num_of_samples = GetAudioSamplesCount();
343 
344  // Resample to new sample rate (if needed)
345  if (new_sample_rate != sample_rate)
346  {
347  // YES, RESAMPLE AUDIO
348  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
349 
350  // Resample data, and return new buffer pointer
351  buffer = resampler->GetResampledBuffer();
352 
353  // Update num_of_samples
354  num_of_samples = buffer->getNumSamples();
355  }
356 
357  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
358  output = new float[num_of_channels * num_of_samples];
359  int position = 0;
360 
361  // Loop through samples in each channel (combining them)
362  for (int channel = 0; channel < num_of_channels; channel++)
363  {
364  for (int sample = 0; sample < num_of_samples; sample++)
365  {
366  // Add sample to output array
367  output[position] = buffer->getReadPointer(channel)[sample];
368 
369  // increment position
370  position++;
371  }
372  }
373 
374  // Update sample count (since it might have changed due to resampling)
375  *sample_count = num_of_samples;
376 
377  // return combined array
378  return output;
379 }
380 
381 
382 // Get an array of sample data (all channels interleaved together), using any sample rate
383 float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
384 {
385  float *output = NULL;
386  AudioSampleBuffer *buffer(audio.get());
387  int num_of_channels = audio->getNumChannels();
388  int num_of_samples = GetAudioSamplesCount();
389 
390  // Resample to new sample rate (if needed)
391  if (new_sample_rate != sample_rate && resampler)
392  {
393  // YES, RESAMPLE AUDIO
394  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
395 
396  // Resample data, and return new buffer pointer
397  buffer = resampler->GetResampledBuffer();
398 
399  // Update num_of_samples
400  num_of_samples = buffer->getNumSamples();
401  }
402 
403  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
404  output = new float[num_of_channels * num_of_samples];
405  int position = 0;
406 
407  // Loop through samples in each channel (combining them)
408  for (int sample = 0; sample < num_of_samples; sample++)
409  {
410  for (int channel = 0; channel < num_of_channels; channel++)
411  {
412  // Add sample to output array
413  output[position] = buffer->getReadPointer(channel)[sample];
414 
415  // increment position
416  position++;
417  }
418  }
419 
420  // Update sample count (since it might have changed due to resampling)
421  *sample_count = num_of_samples;
422 
423  // return combined array
424  return output;
425 }
426 
427 // Get number of audio channels
429 {
430  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
431  if (audio)
432  return audio->getNumChannels();
433  else
434  return 0;
435 }
436 
437 // Get number of audio samples
439 {
440  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
441  return max_audio_sample;
442 }
443 
444 juce::AudioSampleBuffer *Frame::GetAudioSampleBuffer()
445 {
446  return audio.get();
447 }
448 
449 // Get the size in bytes of this frame (rough estimate)
451 {
452  int64_t total_bytes = 0;
453  if (image)
454  total_bytes += (width * height * sizeof(char) * 4);
455  if (audio) {
456  // approximate audio size (sample rate / 24 fps)
457  total_bytes += (sample_rate / 24.0) * sizeof(float);
458  }
459 
460  // return size of this frame
461  return total_bytes;
462 }
463 
464 // Get pixel data (as packets)
465 const unsigned char* Frame::GetPixels()
466 {
467  // Check for blank image
468  if (!image)
469  // Fill with black
470  AddColor(width, height, color);
471 
472  // Return array of pixel packets
473  return image->bits();
474 }
475 
476 // Get pixel data (for only a single scan-line)
477 const unsigned char* Frame::GetPixels(int row)
478 {
479  // Return array of pixel packets
480  return image->scanLine(row);
481 }
482 
483 // Set Pixel Aspect Ratio
484 void Frame::SetPixelRatio(int num, int den)
485 {
486  pixel_ratio.num = num;
487  pixel_ratio.den = den;
488 }
489 
490 // Set frame number
491 void Frame::SetFrameNumber(int64_t new_number)
492 {
493  number = new_number;
494 }
495 
496 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
497 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
498 {
499  // Get the total # of samples for the previous frame, and the current frame (rounded)
500  double fps_rate = fps.Reciprocal().ToDouble();
501 
502  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
503  double previous_samples = (sample_rate * fps_rate) * (number - 1);
504  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
505  previous_samples -= previous_samples_remainder;
506 
507  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
508  double total_samples = (sample_rate * fps_rate) * number;
509  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
510  total_samples -= total_samples_remainder;
511 
512  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
513  // be evenly divided into frames, so each frame can have have different # of samples.
514  int samples_per_frame = round(total_samples - previous_samples);
515  if (samples_per_frame < 0)
516  samples_per_frame = 0;
517  return samples_per_frame;
518 }
519 
520 // Calculate the # of samples per video frame (for the current frame number)
521 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
522 {
523  return GetSamplesPerFrame(number, fps, sample_rate, channels);
524 }
525 
526 // Get height of image
528 {
529  return height;
530 }
531 
532 // Get height of image
534 {
535  return width;
536 }
537 
538 // Get the original sample rate of this frame's audio data
540 {
541  return sample_rate;
542 }
543 
544 // Get the original sample rate of this frame's audio data
546 {
547  return channel_layout;
548 }
549 
550 
551 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
552 void Frame::Save(string path, float scale, string format, int quality)
553 {
554  // Get preview image
555  std::shared_ptr<QImage> previewImage = GetImage();
556 
557  // scale image if needed
558  if (abs(scale) > 1.001 || abs(scale) < 0.999)
559  {
560  int new_width = width;
561  int new_height = height;
562 
563  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
564  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
565  {
566  // Calculate correct DAR (display aspect ratio)
567  int new_width = previewImage->size().width();
568  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
569 
570  // Resize to fix DAR
571  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
572  }
573 
574  // Resize image
575  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width * scale, new_height * scale, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
576  }
577 
578  // Save image
579  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
580 }
581 
582 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
583 void Frame::Thumbnail(string path, int new_width, int new_height, string mask_path, string overlay_path,
584  string background_color, bool ignore_aspect, string format, int quality, float rotate) {
585 
586  // Create blank thumbnail image & fill background color
587  std::shared_ptr<QImage> thumbnail = std::shared_ptr<QImage>(new QImage(new_width, new_height, QImage::Format_RGBA8888));
588  thumbnail->fill(QColor(QString::fromStdString(background_color)));
589 
590  // Create painter
591  QPainter painter(thumbnail.get());
592  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
593 
594  // Get preview image
595  std::shared_ptr<QImage> previewImage = GetImage();
596 
597  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
598  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
599  {
600  // Calculate correct DAR (display aspect ratio)
601  int aspect_width = previewImage->size().width();
602  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
603 
604  // Resize to fix DAR
605  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(aspect_width, aspect_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
606  }
607 
608  // Resize frame image
609  if (ignore_aspect)
610  // Ignore aspect ratio
611  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
612  else
613  // Maintain aspect ratio
614  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
615 
616  // Composite frame image onto background (centered)
617  int x = (new_width - previewImage->size().width()) / 2.0; // center
618  int y = (new_height - previewImage->size().height()) / 2.0; // center
619  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
620 
621 
622  // Create transform and rotate (if needed)
623  QTransform transform;
624  float origin_x = previewImage->width() / 2.0;
625  float origin_y = previewImage->height() / 2.0;
626  transform.translate(origin_x, origin_y);
627  transform.rotate(rotate);
628  transform.translate(-origin_x,-origin_y);
629  painter.setTransform(transform);
630 
631  // Draw image onto QImage
632  painter.drawImage(x, y, *previewImage);
633 
634 
635  // Overlay Image (if any)
636  if (overlay_path != "") {
637  // Open overlay
638  std::shared_ptr<QImage> overlay = std::shared_ptr<QImage>(new QImage());
639  overlay->load(QString::fromStdString(overlay_path));
640 
641  // Set pixel format
642  overlay = std::shared_ptr<QImage>(new QImage(overlay->convertToFormat(QImage::Format_RGBA8888)));
643 
644  // Resize to fit
645  overlay = std::shared_ptr<QImage>(new QImage(overlay->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
646 
647  // Composite onto thumbnail
648  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
649  painter.drawImage(0, 0, *overlay);
650  }
651 
652 
653  // Mask Image (if any)
654  if (mask_path != "") {
655  // Open mask
656  std::shared_ptr<QImage> mask = std::shared_ptr<QImage>(new QImage());
657  mask->load(QString::fromStdString(mask_path));
658 
659  // Set pixel format
660  mask = std::shared_ptr<QImage>(new QImage(mask->convertToFormat(QImage::Format_RGBA8888)));
661 
662  // Resize to fit
663  mask = std::shared_ptr<QImage>(new QImage(mask->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
664 
665  // Negate mask
666  mask->invertPixels();
667 
668  // Get pixels
669  unsigned char *pixels = (unsigned char *) thumbnail->bits();
670  unsigned char *mask_pixels = (unsigned char *) mask->bits();
671 
672  // Convert the mask image to grayscale
673  // Loop through pixels
674  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
675  {
676  // Get the RGB values from the pixel
677  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
678  int Frame_Alpha = pixels[byte_index + 3];
679  int Mask_Value = constrain(Frame_Alpha - gray_value);
680 
681  // Set all alpha pixels to gray value
682  pixels[byte_index + 3] = Mask_Value;
683  }
684  }
685 
686 
687  // End painter
688  painter.end();
689 
690  // Save image
691  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
692 }
693 
694 // Constrain a color value from 0 to 255
695 int Frame::constrain(int color_value)
696 {
697  // Constrain new color from 0 to 255
698  if (color_value < 0)
699  color_value = 0;
700  else if (color_value > 255)
701  color_value = 255;
702 
703  return color_value;
704 }
705 
706 // Add (or replace) pixel data to the frame (based on a solid color)
707 void Frame::AddColor(int new_width, int new_height, string new_color)
708 {
709  // Set color
710  color = new_color;
711 
712  // Create new image object, and fill with pixel data
713  const GenericScopedLock<CriticalSection> lock(addingImageSection);
714  #pragma omp critical (AddImage)
715  {
716  image = std::shared_ptr<QImage>(new QImage(new_width, new_height, QImage::Format_RGBA8888));
717 
718  // Fill with solid color
719  image->fill(QColor(QString::fromStdString(color)));
720  }
721  // Update height and width
722  width = image->width();
723  height = image->height();
724  has_image_data = true;
725 }
726 
727 // Add (or replace) pixel data to the frame
728 void Frame::AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
729 {
730  // Create new buffer
731  const GenericScopedLock<CriticalSection> lock(addingImageSection);
732  int buffer_size = new_width * new_height * bytes_per_pixel;
733  qbuffer = new unsigned char[buffer_size]();
734 
735  // Copy buffer data
736  memcpy((unsigned char*)qbuffer, pixels_, buffer_size);
737 
738  // Create new image object, and fill with pixel data
739  #pragma omp critical (AddImage)
740  {
741  image = std::shared_ptr<QImage>(new QImage(qbuffer, new_width, new_height, new_width * bytes_per_pixel, type, (QImageCleanupFunction) &openshot::Frame::cleanUpBuffer, (void*) qbuffer));
742 
743  // Always convert to RGBA8888 (if different)
744  if (image->format() != QImage::Format_RGBA8888)
745  *image = image->convertToFormat(QImage::Format_RGBA8888);
746 
747  // Update height and width
748  width = image->width();
749  height = image->height();
750  has_image_data = true;
751  }
752 }
753 
754 // Add (or replace) pixel data to the frame
755 void Frame::AddImage(std::shared_ptr<QImage> new_image)
756 {
757  // Ignore blank images
758  if (!new_image)
759  return;
760 
761  // assign image data
762  const GenericScopedLock<CriticalSection> lock(addingImageSection);
763  #pragma omp critical (AddImage)
764  {
765  image = new_image;
766 
767  // Always convert to RGBA8888 (if different)
768  if (image->format() != QImage::Format_RGBA8888)
769  *image = image->convertToFormat(QImage::Format_RGBA8888);
770 
771  // Update height and width
772  width = image->width();
773  height = image->height();
774  has_image_data = true;
775  }
776 }
777 
778 // Add (or replace) pixel data to the frame (for only the odd or even lines)
779 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
780 {
781  // Ignore blank new_image
782  if (!new_image)
783  return;
784 
785  // Check for blank source image
786  if (!image) {
787  // Replace the blank source image
788  AddImage(new_image);
789 
790  } else {
791 
792  // Ignore image of different sizes or formats
793  bool ret=false;
794  #pragma omp critical (AddImage)
795  if (image == new_image || image->size() != image->size() || image->format() != image->format())
796  ret=true;
797  if (ret)
798  return;
799 
800  // Get the frame's image
801  const GenericScopedLock<CriticalSection> lock(addingImageSection);
802  #pragma omp critical (AddImage)
803  {
804  const unsigned char *pixels = image->bits();
805  const unsigned char *new_pixels = new_image->bits();
806 
807  // Loop through the scanlines of the image (even or odd)
808  int start = 0;
809  if (only_odd_lines)
810  start = 1;
811  for (int row = start; row < image->height(); row += 2) {
812  memcpy((unsigned char *) pixels, new_pixels + (row * image->bytesPerLine()), image->bytesPerLine());
813  new_pixels += image->bytesPerLine();
814  }
815 
816  // Update height and width
817  width = image->width();
818  height = image->height();
819  has_image_data = true;
820  }
821  }
822 }
823 
824 
825 // Resize audio container to hold more (or less) samples and channels
826 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
827 {
828  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
829 
830  // Resize JUCE audio buffer
831  audio->setSize(channels, length, true, true, false);
832  channel_layout = layout;
833  sample_rate = rate;
834 
835  // Calculate max audio sample added
836  max_audio_sample = length;
837 }
838 
839 // Add audio samples to a specific channel
840 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
841  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
842  #pragma omp critical (adding_audio)
843  {
844  // Clamp starting sample to 0
845  int destStartSampleAdjusted = max(destStartSample, 0);
846 
847  // Extend audio container to hold more (or less) samples and channels.. if needed
848  int new_length = destStartSampleAdjusted + numSamples;
849  int new_channel_length = audio->getNumChannels();
850  if (destChannel >= new_channel_length)
851  new_channel_length = destChannel + 1;
852  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
853  audio->setSize(new_channel_length, new_length, true, true, false);
854 
855  // Clear the range of samples first (if needed)
856  if (replaceSamples)
857  audio->clear(destChannel, destStartSampleAdjusted, numSamples);
858 
859  // Add samples to frame's audio buffer
860  audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
861  has_audio_data = true;
862 
863  // Calculate max audio sample added
864  if (new_length > max_audio_sample)
865  max_audio_sample = new_length;
866  }
867 }
868 
869 // Apply gain ramp (i.e. fading volume)
870 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
871 {
872  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
873 
874  // Apply gain ramp
875  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
876 }
877 
878 // Get pointer to Magick++ image object
879 std::shared_ptr<QImage> Frame::GetImage()
880 {
881  // Check for blank image
882  if (!image)
883  // Fill with black
884  AddColor(width, height, color);
885 
886  return image;
887 }
888 
889 #ifdef USE_IMAGEMAGICK
890 // Get pointer to ImageMagick image object
891 std::shared_ptr<Magick::Image> Frame::GetMagickImage()
892 {
893  // Check for blank image
894  if (!image)
895  // Fill with black
896  AddColor(width, height, "#000000");
897 
898  // Get the pixels from the frame image
899  QRgb const *tmpBits = (const QRgb*)image->bits();
900 
901  // Create new image object, and fill with pixel data
902  std::shared_ptr<Magick::Image> magick_image = std::shared_ptr<Magick::Image>(new Magick::Image(image->width(), image->height(),"RGBA", Magick::CharPixel, tmpBits));
903 
904  // Give image a transparent background color
905  magick_image->backgroundColor(Magick::Color("none"));
906  magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
907  magick_image->matte(true);
908 
909  return magick_image;
910 }
911 #endif
912 
913 #ifdef USE_IMAGEMAGICK
914 // Get pointer to QImage of frame
915 void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
916 {
917  const int BPP = 4;
918  const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
919 
920  /// Use realloc for fast memory allocation.
921  /// TODO: consider locking the buffer for mt safety
922  //qbuffer = reinterpret_cast<unsigned char*>(realloc(qbuffer, bufferSize));
923  qbuffer = new unsigned char[bufferSize]();
924  unsigned char *buffer = (unsigned char*)qbuffer;
925 
926  // Iterate through the pixel packets, and load our own buffer
927  // Each color needs to be scaled to 8 bit (using the ImageMagick built-in ScaleQuantumToChar function)
928  int numcopied = 0;
929  Magick::PixelPacket *pixels = new_image->getPixels(0,0, new_image->columns(), new_image->rows());
930  for (int n = 0, i = 0; n < new_image->columns() * new_image->rows(); n += 1, i += 4) {
931  buffer[i+0] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].red);
932  buffer[i+1] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].green);
933  buffer[i+2] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].blue);
934  buffer[i+3] = 255 - MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].opacity);
935  numcopied+=4;
936  }
937 
938  // Create QImage of frame data
939  image = std::shared_ptr<QImage>(new QImage(qbuffer, width, height, width * BPP, QImage::Format_RGBA8888, (QImageCleanupFunction) &cleanUpBuffer, (void*) qbuffer));
940 
941  // Update height and width
942  width = image->width();
943  height = image->height();
944  has_image_data = true;
945 }
946 #endif
947 
948 // Play audio samples for this frame
950 {
951  // Check if samples are present
952  if (!GetAudioSamplesCount())
953  return;
954 
955  AudioDeviceManager deviceManager;
956  deviceManager.initialise (0, /* number of input channels */
957  2, /* number of output channels */
958  0, /* no XML settings.. */
959  true /* select default device on failure */);
960  //deviceManager.playTestSound();
961 
962  AudioSourcePlayer audioSourcePlayer;
963  deviceManager.addAudioCallback (&audioSourcePlayer);
964 
965  ScopedPointer<AudioBufferSource> my_source;
966  my_source = new AudioBufferSource(audio.get());
967 
968  // Create TimeSliceThread for audio buffering
969  TimeSliceThread my_thread("Audio buffer thread");
970 
971  // Start thread
972  my_thread.startThread();
973 
974  AudioTransportSource transport1;
975  transport1.setSource (my_source,
976  5000, // tells it to buffer this many samples ahead
977  &my_thread,
978  (double) sample_rate,
979  audio->getNumChannels()); // sample rate of source
980  transport1.setPosition (0);
981  transport1.setGain(1.0);
982 
983 
984  // Create MIXER
985  MixerAudioSource mixer;
986  mixer.addInputSource(&transport1, false);
987  audioSourcePlayer.setSource (&mixer);
988 
989  // Start transports
990  transport1.start();
991 
992  while (transport1.isPlaying())
993  {
994  cout << "playing" << endl;
995  usleep(1000000);
996  }
997 
998  cout << "DONE!!!" << endl;
999 
1000  transport1.stop();
1001  transport1.setSource (0);
1002  audioSourcePlayer.setSource (0);
1003  my_thread.stopThread(500);
1004  deviceManager.removeAudioCallback (&audioSourcePlayer);
1005  deviceManager.closeAudioDevice();
1006  deviceManager.removeAllChangeListeners();
1007  deviceManager.dispatchPendingMessages();
1008 
1009  cout << "End of Play()" << endl;
1010 
1011 
1012 }
1013 
1014 // Clean up buffer after QImage is deleted
1015 void Frame::cleanUpBuffer(void *info)
1016 {
1017  if (info)
1018  {
1019  // Remove buffer since QImage tells us to
1020  unsigned char* ptr_to_qbuffer = (unsigned char*) info;
1021  delete[] ptr_to_qbuffer;
1022  }
1023 }
1024 
1025 // Add audio silence
1026 void Frame::AddAudioSilence(int numSamples)
1027 {
1028  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
1029 
1030  // Resize audio container
1031  audio->setSize(channels, numSamples, false, true, false);
1032  audio->clear();
1033  has_audio_data = true;
1034 
1035  // Calculate max audio sample added
1036  if (numSamples > max_audio_sample)
1037  max_audio_sample = numSamples;
1038 }
void SetBuffer(AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
int GetWidth()
Get height of image.
Definition: Frame.cpp:533
int num
Numerator for the fraction.
Definition: Fraction.h:44
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:438
float * GetInterleavedAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
Definition: Frame.cpp:383
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:115
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:274
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:444
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:465
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:949
void Save(string path, float scale, string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
Definition: Frame.cpp:552
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:102
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:840
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:131
Fraction Reciprocal()
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:81
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:138
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:93
void ClearWaveform()
Clear the waveform image (and deallocate it's memory)
Definition: Frame.cpp:267
void AddColor(int new_width, int new_height, string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:707
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:330
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:491
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1026
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:139
This class represents a fraction.
Definition: Fraction.h:42
void Thumbnail(string path, int new_width, int new_height, string mask_path, string overlay_path, string background_color, bool ignore_aspect, string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:583
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:1015
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:728
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
Definition: Frame.cpp:34
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:870
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:284
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:484
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:428
ChannelLayout ChannelsLayout()
Definition: Frame.cpp:545
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:879
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:172
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:450
~Frame()
Destructor.
Definition: Frame.cpp:124
AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
float * GetPlanarAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:337
int den
Denominator for the fraction.
Definition: Fraction.h:45
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
Definition: Frame.cpp:317
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:521
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:140
void ResizeAudio(int channels, int length, int sample_rate, ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:826
int GetHeight()
Get height of image.
Definition: Frame.cpp:527
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:539
This class is used to resample audio data for many sequential frames.