OpenShot Library | libopenshot  0.1.2
FrameMapper.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for the FrameMapper class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2014 OpenShot Studios, LLC
9  * <http://www.openshotstudios.com/>. This file is part of
10  * OpenShot Library (libopenshot), an open-source project dedicated to
11  * delivering high quality video editing and animation solutions to the
12  * world. For more information visit <http://www.openshot.org/>.
13  *
14  * OpenShot Library (libopenshot) is free software: you can redistribute it
15  * and/or modify it under the terms of the GNU Lesser General Public License
16  * as published by the Free Software Foundation, either version 3 of the
17  * License, or (at your option) any later version.
18  *
19  * OpenShot Library (libopenshot) is distributed in the hope that it will be
20  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22  * GNU Lesser General Public License for more details.
23  *
24  * You should have received a copy of the GNU Lesser General Public License
25  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
26  */
27 
28 #include "../include/FrameMapper.h"
29 
30 using namespace std;
31 using namespace openshot;
32 
33 FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
34  reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
35 {
36  // Set the original frame rate from the reader
37  original = Fraction(reader->info.fps.num, reader->info.fps.den);
38 
39  // Set all info struct members equal to the internal reader
40  info = reader->info;
41  info.fps.num = target.num;
42  info.fps.den = target.den;
43  info.video_timebase.num = target.den;
44  info.video_timebase.den = target.num;
46  info.sample_rate = target_sample_rate;
47  info.channels = target_channels;
48  info.channel_layout = target_channel_layout;
49  info.width = reader->info.width;
50  info.height = reader->info.height;
51 
52  // Used to toggle odd / even fields
53  field_toggle = true;
54 
55  // Adjust cache size based on size of frame and audio
57 
58  // init mapping between original and target frames
59  Init();
60 }
61 
62 // Destructor
64  if (is_open)
65  // Auto Close if not already
66  Close();
67 }
68 
69 void FrameMapper::AddField(long int frame)
70 {
71  // Add a field, and toggle the odd / even field
72  AddField(Field(frame, field_toggle));
73 }
74 
75 void FrameMapper::AddField(Field field)
76 {
77  // Add a field to the end of the field list
78  fields.push_back(field);
79 
80  // toggle the odd / even flag
81  field_toggle = (field_toggle ? false : true);
82 }
83 
84 // Use the original and target frame rates and a pull-down technique to create
85 // a mapping between the original fields and frames or a video to a new frame rate.
86 // This might repeat or skip fields and frames of the original video, depending on
87 // whether the frame rate is increasing or decreasing.
88 void FrameMapper::Init()
89 {
90  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
91 
92  // Do not initialize anything if just a picture with no audio
94  // Skip initialization
95  return;
96 
97  // Clear the fields & frames lists
98  fields.clear();
99  frames.clear();
100 
101  // Mark as not dirty
102  is_dirty = false;
103 
104  // Clear cache
105  final_cache.Clear();
106 
107  // Some framerates are handled special, and some use a generic Keyframe curve to
108  // map the framerates. These are the special framerates:
109  if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
110  (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
111 
112  // Get the difference (in frames) between the original and target frame rates
113  float difference = target.ToInt() - original.ToInt();
114 
115  // Find the number (i.e. interval) of fields that need to be skipped or repeated
116  int field_interval = 0;
117  int frame_interval = 0;
118 
119  if (difference != 0)
120  {
121  field_interval = round(fabs(original.ToInt() / difference));
122 
123  // Get frame interval (2 fields per frame)
124  frame_interval = field_interval * 2.0f;
125  }
126 
127 
128  // Calculate # of fields to map
129  long int frame = 1;
130  long int number_of_fields = reader->info.video_length * 2;
131 
132  // Loop through all fields in the original video file
133  for (long int field = 1; field <= number_of_fields; field++)
134  {
135 
136  if (difference == 0) // Same frame rate, NO pull-down or special techniques required
137  {
138  // Add fields
139  AddField(frame);
140  }
141  else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
142  {
143  // Add current field
144  AddField(frame);
145 
146  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
147  {
148  // Add extra field for each 'field interval
149  AddField(frame);
150  }
151  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
152  {
153  // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
154  AddField(frame); // add field for current frame
155 
156  if (frame + 1 <= info.video_length)
157  // add field for next frame (if the next frame exists)
158  AddField(Field(frame + 1, field_toggle));
159  }
160  else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
161  {
162  // No pull-down technique needed, just repeat this frame
163  AddField(frame);
164  AddField(frame);
165  }
166  }
167  else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
168  {
169 
170  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
171  {
172  // skip current field and toggle the odd/even flag
173  field_toggle = (field_toggle ? false : true);
174  }
175  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
176  {
177  // skip this field, plus the next field
178  field++;
179  }
180  else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
181  {
182  // skip this field, plus the next one
183  field++;
184  }
185  else
186  {
187  // No skipping needed, so add the field
188  AddField(frame);
189  }
190  }
191 
192  // increment frame number (if field is divisible by 2)
193  if (field % 2 == 0 && field > 0)
194  frame++;
195  }
196 
197  } else {
198  // Map the remaining framerates using a simple Keyframe curve
199  // Calculate the difference (to be used as a multiplier)
200  float rate_diff = target.ToFloat() / original.ToFloat();
201  long int new_length = reader->info.video_length * rate_diff;
202 
203  // Build curve for framerate mapping
204  Keyframe rate_curve;
205  rate_curve.AddPoint(1, 1, LINEAR);
206  rate_curve.AddPoint(new_length, reader->info.video_length, LINEAR);
207 
208  // Loop through curve, and build list of frames
209  for (long int frame_num = 1; frame_num <= new_length; frame_num++)
210  {
211  // Add 2 fields per frame
212  AddField(rate_curve.GetInt(frame_num));
213  AddField(rate_curve.GetInt(frame_num));
214  }
215  }
216 
217  // Loop through the target frames again (combining fields into frames)
218  Field Odd(0, true); // temp field used to track the ODD field
219  Field Even(0, true); // temp field used to track the EVEN field
220 
221  // Variables used to remap audio samples
222  int start_samples_frame = 1;
223  int start_samples_position = 0;
224 
225  for (long int field = 1; field <= fields.size(); field++)
226  {
227  // Get the current field
228  Field f = fields[field - 1];
229 
230  // Is field divisible by 2?
231  if (field % 2 == 0 && field > 0)
232  {
233  // New frame number
234  long int frame_number = field / 2;
235 
236  // Set the bottom frame
237  if (f.isOdd)
238  Odd = f;
239  else
240  Even = f;
241 
242  // Determine the range of samples (from the original rate). Resampling happens in real-time when
243  // calling the GetFrame() method. So this method only needs to redistribute the original samples with
244  // the original sample rate.
245  int end_samples_frame = start_samples_frame;
246  int end_samples_position = start_samples_position;
247  int remaining_samples = Frame::GetSamplesPerFrame(frame_number, target, reader->info.sample_rate, reader->info.channels);
248 
249  while (remaining_samples > 0)
250  {
251  // get original samples
252  int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
253 
254  // Enough samples
255  if (original_samples >= remaining_samples)
256  {
257  // Take all that we need, and break loop
258  end_samples_position += remaining_samples;
259  remaining_samples = 0;
260  } else
261  {
262  // Not enough samples (take them all, and keep looping)
263  end_samples_frame += 1; // next frame
264  end_samples_position = 0; // next frame, starting on 1st sample
265  remaining_samples -= original_samples; // reduce the remaining amount
266  }
267  }
268 
269 
270 
271  // Create the sample mapping struct
272  SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(frame_number, target, reader->info.sample_rate, reader->info.channels)};
273 
274  // Reset the audio variables
275  start_samples_frame = end_samples_frame;
276  start_samples_position = end_samples_position + 1;
277  if (start_samples_position >= Frame::GetSamplesPerFrame(start_samples_frame, original, reader->info.sample_rate, reader->info.channels))
278  {
279  start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
280  start_samples_position = 0; // reset to 0, since we wrapped
281  }
282 
283  // Create a frame and ADD it to the frames collection
284  MappedFrame frame = {Odd, Even, Samples};
285  frames.push_back(frame);
286  }
287  else
288  {
289  // Set the top field
290  if (f.isOdd)
291  Odd = f;
292  else
293  Even = f;
294  }
295  }
296 
297  // Clear the internal fields list (no longer needed)
298  fields.clear();
299 }
300 
301 MappedFrame FrameMapper::GetMappedFrame(long int TargetFrameNumber) throw(OutOfBoundsFrame)
302 {
303  // Ignore mapping on single image readers
305  // Return the same number
306  MappedFrame frame;
307  frame.Even.Frame = TargetFrameNumber;
308  frame.Odd.Frame = TargetFrameNumber;
309  frame.Samples.frame_start = 0;
310  frame.Samples.frame_end = 0;
311  frame.Samples.sample_start = 0;
312  frame.Samples.sample_end = 0;
313  frame.Samples.total = 0;
314  return frame;
315  }
316 
317  // Check if frame number is valid
318  if(TargetFrameNumber < 1 || frames.size() == 0)
319  // frame too small, return error
320  throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
321 
322  else if (TargetFrameNumber > frames.size())
323  // frame too large, set to end frame
324  TargetFrameNumber = frames.size();
325 
326  // Debug output
327  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetMappedFrame", "TargetFrameNumber", TargetFrameNumber, "frames.size()", frames.size(), "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame, "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame, "", -1, "", -1);
328 
329  // Return frame
330  return frames[TargetFrameNumber - 1];
331 }
332 
333 // Get or generate a blank frame
334 tr1::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(long int number)
335 {
336  tr1::shared_ptr<Frame> new_frame;
337 
338  // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
339  int samples_in_frame = Frame::GetSamplesPerFrame(number, target, reader->info.sample_rate, reader->info.channels);
340 
341  try {
342  // Debug output
343  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (from reader)", "number", number, "samples_in_frame", samples_in_frame, "", -1, "", -1, "", -1, "", -1);
344 
345  // Attempt to get a frame (but this could fail if a reader has just been closed)
346  new_frame = reader->GetFrame(number);
347 
348  // Return real frame
349  return new_frame;
350 
351  } catch (const ReaderClosed & e) {
352  // ...
353  } catch (const TooManySeeks & e) {
354  // ...
355  } catch (const OutOfBoundsFrame & e) {
356  // ...
357  }
358 
359  // Debug output
360  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (create blank)", "number", number, "samples_in_frame", samples_in_frame, "", -1, "", -1, "", -1, "", -1);
361 
362  // Create blank frame
363  new_frame = tr1::shared_ptr<Frame>(new Frame(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels));
364  new_frame->SampleRate(reader->info.sample_rate);
365  new_frame->ChannelsLayout(info.channel_layout);
366  return new_frame;
367 }
368 
369 // Get an openshot::Frame object for a specific frame number of this reader.
370 tr1::shared_ptr<Frame> FrameMapper::GetFrame(long int requested_frame) throw(ReaderClosed)
371 {
372  // Check final cache, and just return the frame (if it's available)
373  tr1::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
374  if (final_frame) return final_frame;
375 
376  // Create a scoped lock, allowing only a single thread to run the following code at one time
377  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
378 
379  // Check if mappings are dirty (and need to be recalculated)
380  if (is_dirty)
381  // Recalculate mappings
382  Init();
383 
384  // Check final cache a 2nd time (due to potential lock already generating this frame)
385  final_frame = final_cache.GetFrame(requested_frame);
386  if (final_frame) return final_frame;
387 
388  // Minimum number of frames to process (for performance reasons)
389  int minimum_frames = OPEN_MP_NUM_PROCESSORS;
390 
391  // Set the number of threads in OpenMP
392  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
393  // Allow nested OpenMP sections
394  omp_set_nested(true);
395 
396  // Debug output
397  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (Loop through frames)", "requested_frame", requested_frame, "minimum_frames", minimum_frames, "", -1, "", -1, "", -1, "", -1);
398 
399  #pragma omp parallel
400  {
401  // Loop through all requested frames, each frame gets it's own thread
402  #pragma omp for ordered firstprivate(requested_frame, minimum_frames)
403  for (long int frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
404  {
405 
406  // Debug output
407  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (inside omp for loop)", "frame_number", frame_number, "minimum_frames", minimum_frames, "requested_frame", requested_frame, "", -1, "", -1, "", -1);
408 
409  // Get the mapped frame
410  MappedFrame mapped = GetMappedFrame(frame_number);
411  tr1::shared_ptr<Frame> mapped_frame;
412 
413  // Get the mapped frame (keeping the sample rate and channels the same as the original... for the moment)
414  mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
415 
416  // Get # of channels in the actual frame
417  int channels_in_frame = mapped_frame->GetAudioChannelsCount();
418  int samples_in_frame = Frame::GetSamplesPerFrame(frame_number, target, mapped_frame->SampleRate(), channels_in_frame);
419 
420  // Determine if mapped frame is identical to source frame
421  if (info.sample_rate == mapped_frame->SampleRate() &&
422  info.channels == mapped_frame->GetAudioChannelsCount() &&
423  info.channel_layout == mapped_frame->ChannelsLayout() &&
424  info.fps.num == reader->info.fps.num &&
425  info.fps.den == reader->info.fps.den) {
426  // Add original frame to cache, and skip the rest (for performance reasons)
427  final_cache.Add(frame_number, mapped_frame);
428  continue;
429  }
430 
431  // Create a new frame
432  tr1::shared_ptr<Frame> frame = tr1::shared_ptr<Frame>(new Frame(frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame));
433  frame->SampleRate(mapped_frame->SampleRate());
434  frame->ChannelsLayout(mapped_frame->ChannelsLayout());
435 
436 
437  // Copy the image from the odd field
438  tr1::shared_ptr<Frame> odd_frame;
439  #pragma omp ordered
440  odd_frame = GetOrCreateFrame(mapped.Odd.Frame);
441 
442  if (odd_frame)
443  frame->AddImage(tr1::shared_ptr<QImage>(new QImage(*odd_frame->GetImage())), true);
444  if (mapped.Odd.Frame != mapped.Even.Frame) {
445  // Add even lines (if different than the previous image)
446  tr1::shared_ptr<Frame> even_frame;
447  #pragma omp ordered
448  even_frame = GetOrCreateFrame(mapped.Even.Frame);
449  if (even_frame)
450  frame->AddImage(tr1::shared_ptr<QImage>(new QImage(*even_frame->GetImage())), false);
451  }
452 
453  // Copy the samples
454  int samples_copied = 0;
455  int starting_frame = mapped.Samples.frame_start;
456  while (info.has_audio && samples_copied < mapped.Samples.total)
457  {
458  // Init number of samples to copy this iteration
459  int remaining_samples = mapped.Samples.total - samples_copied;
460  int number_to_copy = 0;
461 
462  // Loop through each channel
463  for (int channel = 0; channel < channels_in_frame; channel++)
464  {
465  // number of original samples on this frame
466  tr1::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
467  int original_samples = original_frame->GetAudioSamplesCount();
468 
469  if (starting_frame == mapped.Samples.frame_start)
470  {
471  // Starting frame (take the ending samples)
472  number_to_copy = original_samples - mapped.Samples.sample_start;
473  if (number_to_copy > remaining_samples)
474  number_to_copy = remaining_samples;
475 
476  // Add samples to new frame
477  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.Samples.sample_start, number_to_copy, 1.0);
478  }
479  else if (starting_frame > mapped.Samples.frame_start && starting_frame < mapped.Samples.frame_end)
480  {
481  // Middle frame (take all samples)
482  number_to_copy = original_samples;
483  if (number_to_copy > remaining_samples)
484  number_to_copy = remaining_samples;
485 
486  // Add samples to new frame
487  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
488  }
489  else
490  {
491  // Ending frame (take the beginning samples)
492  number_to_copy = mapped.Samples.sample_end;
493  if (number_to_copy > remaining_samples)
494  number_to_copy = remaining_samples;
495 
496  // Add samples to new frame
497  frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
498  }
499  }
500 
501  // increment frame
502  samples_copied += number_to_copy;
503  starting_frame++;
504  }
505 
506  // Resample audio on frame (if needed)
507  if (info.has_audio &&
508  (info.sample_rate != frame->SampleRate() ||
509  info.channels != frame->GetAudioChannelsCount() ||
510  info.channel_layout != frame->ChannelsLayout()))
511  // Resample audio and correct # of channels if needed
512  ResampleMappedAudio(frame, mapped.Odd.Frame);
513 
514  // Add frame to final cache
515  final_cache.Add(frame->number, frame);
516 
517  } // for loop
518  } // omp parallel
519 
520  // Return processed openshot::Frame
521  return final_cache.GetFrame(requested_frame);
522 }
523 
525 {
526  // Check if mappings are dirty (and need to be recalculated)
527  if (is_dirty)
528  // Recalculate mappings
529  Init();
530 
531  // Get the difference (in frames) between the original and target frame rates
532  float difference = target.ToInt() - original.ToInt();
533 
534  int field_interval = 0;
535  int frame_interval = 0;
536 
537  if (difference != 0)
538  {
539  // Find the number (i.e. interval) of fields that need to be skipped or repeated
540  field_interval = round(fabs(original.ToInt() / difference));
541 
542  // Get frame interval (2 fields per frame)
543  frame_interval = field_interval * 2.0f;
544  }
545 
546  // Loop through frame mappings
547  for (float map = 1; map <= frames.size(); map++)
548  {
549  MappedFrame frame = frames[map - 1];
550  cout << "Target frame #: " << map << " mapped to original frame #:\t(" << frame.Odd.Frame << " odd, " << frame.Even.Frame << " even)" << endl;
551  cout << " - Audio samples mapped to frame " << frame.Samples.frame_start << ":" << frame.Samples.sample_start << " to frame " << frame.Samples.frame_end << ":" << frame.Samples.sample_end << endl;
552  }
553 
554 }
555 
556 
557 // Determine if reader is open or closed
559  if (reader)
560  return reader->IsOpen();
561  else
562  return false;
563 }
564 
565 
566 // Open the internal reader
568 {
569  if (reader)
570  {
571  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
572 
573  // Open the reader
574  reader->Open();
575  }
576 }
577 
578 // Close the internal reader
580 {
581  if (reader)
582  {
583  // Create a scoped lock, allowing only a single thread to run the following code at one time
584  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
585 
586  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
587 
588  // Close internal reader
589  reader->Close();
590 
591  // Deallocate resample buffer
592  if (avr) {
593  avresample_close(avr);
594  avresample_free(&avr);
595  avr = NULL;
596  }
597  }
598 }
599 
600 
601 // Generate JSON string of this object
603 
604  // Return formatted string
605  return JsonValue().toStyledString();
606 }
607 
608 // Generate Json::JsonValue for this object
609 Json::Value FrameMapper::JsonValue() {
610 
611  // Create root json object
612  Json::Value root = ReaderBase::JsonValue(); // get parent properties
613  root["type"] = "FrameMapper";
614 
615  // return JsonValue
616  return root;
617 }
618 
619 // Load JSON string into this object
620 void FrameMapper::SetJson(string value) throw(InvalidJSON) {
621 
622  // Parse JSON string into JSON objects
623  Json::Value root;
624  Json::Reader reader;
625  bool success = reader.parse( value, root );
626  if (!success)
627  // Raise exception
628  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
629 
630  try
631  {
632  // Set all values that match
633  SetJsonValue(root);
634  }
635  catch (exception e)
636  {
637  // Error parsing JSON (or missing keys)
638  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
639  }
640 }
641 
642 // Load Json::JsonValue into this object
643 void FrameMapper::SetJsonValue(Json::Value root) throw(InvalidFile) {
644 
645  // Set parent data
647 
648  // Re-Open path, and re-init everything (if needed)
649  if (reader) {
650 
651  Close();
652  Open();
653  }
654 }
655 
656 // Change frame rate or audio mapping details
657 void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
658 {
659  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ChangeMapping", "target_fps.num", target_fps.num, "target_fps.den", target_fps.num, "target_pulldown", target_pulldown, "target_sample_rate", target_sample_rate, "target_channels", target_channels, "target_channel_layout", target_channel_layout);
660 
661  // Mark as dirty
662  is_dirty = true;
663 
664  // Update mapping details
665  target = target_fps;
666  pulldown = target_pulldown;
667  info.sample_rate = target_sample_rate;
668  info.channels = target_channels;
669  info.channel_layout = target_channel_layout;
670 
671  // Clear cache
672  final_cache.Clear();
673 
674  // Adjust cache size based on size of frame and audio
676 
677  // Deallocate resample buffer
678  if (avr) {
679  avresample_close(avr);
680  avresample_free(&avr);
681  avr = NULL;
682  }
683 }
684 
685 // Resample audio and map channels (if needed)
686 void FrameMapper::ResampleMappedAudio(tr1::shared_ptr<Frame> frame, long int original_frame_number)
687 {
688  // Init audio buffers / variables
689  int total_frame_samples = 0;
690  int channels_in_frame = frame->GetAudioChannelsCount();
691  int sample_rate_in_frame = frame->SampleRate();
692  int samples_in_frame = frame->GetAudioSamplesCount();
693  ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
694 
695  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio", "frame->number", frame->number, "original_frame_number", original_frame_number, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "sample_rate_in_frame", sample_rate_in_frame, "", -1);
696 
697  // Get audio sample array
698  float* frame_samples_float = NULL;
699  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
700  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
701 
702  // Calculate total samples
703  total_frame_samples = samples_in_frame * channels_in_frame;
704 
705  // Create a new array (to hold all S16 audio samples for the current queued frames)
706  int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
707 
708  // Translate audio sample values back to 16 bit integers
709  for (int s = 0; s < total_frame_samples; s++)
710  // Translate sample value and copy into buffer
711  frame_samples[s] = int(frame_samples_float[s] * (1 << 15));
712 
713 
714  // Deallocate float array
715  delete[] frame_samples_float;
716  frame_samples_float = NULL;
717 
718  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (got sample data from frame)", "frame->number", frame->number, "total_frame_samples", total_frame_samples, "target channels", info.channels, "channels_in_frame", channels_in_frame, "target sample_rate", info.sample_rate, "samples_in_frame", samples_in_frame);
719 
720 
721  // Create input frame (and allocate arrays)
722  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
723  AV_RESET_FRAME(audio_frame);
724  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
725 
726  int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
727  audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
728 
729  if (error_code < 0)
730  {
731  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
732  throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
733  }
734 
735  // Update total samples & input frame size (due to bigger or smaller data types)
736  total_frame_samples = Frame::GetSamplesPerFrame(frame->number, target, info.sample_rate, info.channels);
737 
738  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (adjust # of samples)", "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "sample_rate_in_frame", sample_rate_in_frame, "info.channels", info.channels, "channels_in_frame", channels_in_frame, "original_frame_number", original_frame_number);
739 
740  // Create output frame (and allocate arrays)
741  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
742  AV_RESET_FRAME(audio_converted);
743  audio_converted->nb_samples = total_frame_samples;
744  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
745 
746  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (preparing for resample)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", AV_SAMPLE_FMT_S16, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
747 
748  int nb_samples = 0;
749  // Force the audio resampling to happen in order (1st thread to last thread), so the waveform
750  // is smooth and continuous.
751  #pragma omp ordered
752  {
753  // setup resample context
754  if (!avr) {
755  avr = avresample_alloc_context();
756  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
757  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
758  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
759  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
760  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
761  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
762  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
763  av_opt_set_int(avr, "out_channels", info.channels, 0);
764  avresample_open(avr);
765  }
766 
767  // Convert audio samples
768  nb_samples = avresample_convert(avr, // audio resample context
769  audio_converted->data, // output data pointers
770  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
771  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
772  audio_frame->data, // input data pointers
773  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
774  audio_frame->nb_samples); // number of input samples to convert
775  }
776 
777  // Create a new array (to hold all resampled S16 audio samples)
778  int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
779 
780  // Copy audio samples over original samples
781  memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
782 
783  // Free frames
784  av_freep(&audio_frame->data[0]);
785  AV_FREE_FRAME(&audio_frame);
786  av_freep(&audio_converted->data[0]);
787  AV_FREE_FRAME(&audio_converted);
788  frame_samples = NULL;
789 
790  // Resize the frame to hold the right # of channels and samples
791  int channel_buffer_size = nb_samples;
792  frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
793 
794  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Audio successfully resampled)", "nb_samples", nb_samples, "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "channels_in_frame", channels_in_frame, "info.channels", info.channels, "info.channel_layout", info.channel_layout);
795 
796  // Array of floats (to hold samples for each channel)
797  float *channel_buffer = new float[channel_buffer_size];
798 
799  // Divide audio into channels. Loop through each channel
800  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
801  {
802  // Init array
803  for (int z = 0; z < channel_buffer_size; z++)
804  channel_buffer[z] = 0.0f;
805 
806  // Loop through all samples and add them to our Frame based on channel.
807  // Toggle through each channel number, since channel data is stored like (left right left right)
808  int channel = 0;
809  int position = 0;
810  for (int sample = 0; sample < (nb_samples * info.channels); sample++)
811  {
812  // Only add samples for current channel
813  if (channel_filter == channel)
814  {
815  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
816  channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
817 
818  // Increment audio position
819  position++;
820  }
821 
822  // increment channel (if needed)
823  if ((channel + 1) < info.channels)
824  // move to next channel
825  channel ++;
826  else
827  // reset channel
828  channel = 0;
829  }
830 
831  // Add samples to frame for this channel
832  frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
833 
834  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Add audio to channel)", "number of samples", position, "channel_filter", channel_filter, "", -1, "", -1, "", -1, "", -1);
835  }
836 
837  // Update frame's audio meta data
838  frame->SampleRate(info.sample_rate);
839  frame->ChannelsLayout(info.channel_layout);
840 
841  // clear channel buffer
842  delete[] channel_buffer;
843  channel_buffer = NULL;
844 
845  // Delete arrays
846  delete[] resampled_samples;
847  resampled_samples = NULL;
848 }
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
Definition: FrameMapper.h:62
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:44
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
Definition: ReaderBase.h:99
vector< MappedFrame > frames
Definition: FrameMapper.h:167
void Add(long int frame_number, tr1::shared_ptr< Frame > frame)
Add a Frame to the cache.
Definition: Cache.cpp:57
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:83
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:67
tr1::shared_ptr< Frame > GetFrame(long int requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:115
void ResampleMappedAudio(tr1::shared_ptr< Frame > frame, long int original_frame_number)
Resample audio and map channels (if needed)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
~FrameMapper()
Destructor.
Definition: FrameMapper.cpp:63
float duration
Length of time (in seconds)
Definition: ReaderBase.h:64
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:95
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:61
This struct holds a single field (half a frame).
Definition: FrameMapper.h:73
Exception when encoding audio packet.
Definition: Exceptions.h:101
void AddPoint(Point p)
Add a new point on the key-frame. Each point has a primary coordinate, a left handle, and a right handle.
Definition: KeyFrame.cpp:72
This struct holds a the range of samples needed by this frame.
Definition: FrameMapper.h:93
void SetMaxBytesFromInfo(long int number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: Cache.cpp:257
MappedFrame GetMappedFrame(long int TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:62
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int height
The height of the video (in pixels)
Definition: ReaderBase.h:66
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
bool IsOpen()
Determine if reader is open or closed.
long int Frame
Definition: FrameMapper.h:75
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:154
This class represents a fraction.
Definition: Fraction.h:42
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:63
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:104
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:153
#define av_err2str(errnum)
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:108
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:51
This struct holds two fields which together make up a complete video frame.
Definition: FrameMapper.h:110
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:69
vector< Field > fields
Definition: FrameMapper.h:166
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:76
string Json()
Get and Set JSON methods.
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
int GetInt(long int index)
Get the rounded INT value at a specific index.
Definition: KeyFrame.cpp:246
tr1::shared_ptr< Frame > GetFrame(long int frame_number)
Get a frame from the cache.
Definition: Cache.cpp:79
void Clear()
Clear the cache of all frames.
Definition: Cache.cpp:203
This namespace is the default namespace for all code in the openshot library.
virtual tr1::shared_ptr< Frame > GetFrame(long int number)=0
Do not apply pull-down techniques, just repeat or skip entire frames.
Definition: FrameMapper.h:64
Linear curves are angular, straight lines between two points.
Definition: Point.h:47
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
Definition: Exceptions.h:152
PulldownType
This enumeration determines how frame rates are increased or decreased.
Definition: FrameMapper.h:60
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:82
A Keyframe is a collection of Point instances, which is used to vary a number or property over time...
Definition: KeyFrame.h:64
long int video_length
The number of frames in the video stream.
Definition: ReaderBase.h:74
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:497
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
Definition: FrameMapper.h:63
void SetJson(string value)
Load JSON string into this object.
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:81
Exception when too many seek attempts happen.
Definition: Exceptions.h:254
virtual bool IsOpen()=0
A thread safe version of GetFrame.