28 #include "../include/FrameMapper.h" 34 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
69 void FrameMapper::AddField(
long int frame)
72 AddField(
Field(frame, field_toggle));
75 void FrameMapper::AddField(
Field field)
81 field_toggle = (field_toggle ? false :
true);
88 void FrameMapper::Init()
90 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Init (Calculate frame mappings)",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
109 if ((fabs(original.
ToFloat() - 24.0) < 1
e-7 || fabs(original.
ToFloat() - 25.0) < 1
e-7 || fabs(original.
ToFloat() - 30.0) < 1
e-7) &&
110 (fabs(target.
ToFloat() - 24.0) < 1
e-7 || fabs(target.
ToFloat() - 25.0) < 1
e-7 || fabs(target.
ToFloat() - 30.0) < 1
e-7)) {
113 float difference = target.
ToInt() - original.
ToInt();
116 int field_interval = 0;
117 int frame_interval = 0;
121 field_interval = round(fabs(original.
ToInt() / difference));
124 frame_interval = field_interval * 2.0f;
133 for (
long int field = 1; field <= number_of_fields; field++)
141 else if (difference > 0)
151 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
158 AddField(
Field(frame + 1, field_toggle));
160 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
167 else if (difference < 0)
173 field_toggle = (field_toggle ? false :
true);
175 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
180 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
193 if (field % 2 == 0 && field > 0)
209 for (
long int frame_num = 1; frame_num <= new_length; frame_num++)
212 AddField(rate_curve.
GetInt(frame_num));
213 AddField(rate_curve.
GetInt(frame_num));
222 int start_samples_frame = 1;
223 int start_samples_position = 0;
225 for (
long int field = 1; field <=
fields.size(); field++)
231 if (field % 2 == 0 && field > 0)
234 long int frame_number = field / 2;
245 int end_samples_frame = start_samples_frame;
246 int end_samples_position = start_samples_position;
249 while (remaining_samples > 0)
255 if (original_samples >= remaining_samples)
258 end_samples_position += remaining_samples;
259 remaining_samples = 0;
263 end_samples_frame += 1;
264 end_samples_position = 0;
265 remaining_samples -= original_samples;
275 start_samples_frame = end_samples_frame;
276 start_samples_position = end_samples_position + 1;
279 start_samples_frame += 1;
280 start_samples_position = 0;
308 frame.
Odd.
Frame = TargetFrameNumber;
318 if(TargetFrameNumber < 1 ||
frames.size() == 0)
322 else if (TargetFrameNumber >
frames.size())
324 TargetFrameNumber =
frames.size();
327 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetMappedFrame",
"TargetFrameNumber", TargetFrameNumber,
"frames.size()",
frames.size(),
"frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
"frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame,
"", -1,
"", -1);
330 return frames[TargetFrameNumber - 1];
334 tr1::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(
long int number)
336 tr1::shared_ptr<Frame> new_frame;
343 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (from reader)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
346 new_frame = reader->
GetFrame(number);
360 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (create blank)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
373 tr1::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
374 if (final_frame)
return final_frame;
385 final_frame = final_cache.
GetFrame(requested_frame);
386 if (final_frame)
return final_frame;
394 omp_set_nested(
true);
397 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (Loop through frames)",
"requested_frame", requested_frame,
"minimum_frames", minimum_frames,
"", -1,
"", -1,
"", -1,
"", -1);
402 #pragma omp for ordered firstprivate(requested_frame, minimum_frames) 403 for (
long int frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
407 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (inside omp for loop)",
"frame_number", frame_number,
"minimum_frames", minimum_frames,
"requested_frame", requested_frame,
"", -1,
"", -1,
"", -1);
411 tr1::shared_ptr<Frame> mapped_frame;
414 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
417 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
422 info.
channels == mapped_frame->GetAudioChannelsCount() &&
427 final_cache.
Add(frame_number, mapped_frame);
432 tr1::shared_ptr<Frame> frame = tr1::shared_ptr<Frame>(
new Frame(frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame));
433 frame->SampleRate(mapped_frame->SampleRate());
434 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
438 tr1::shared_ptr<Frame> odd_frame;
440 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
443 frame->AddImage(tr1::shared_ptr<QImage>(
new QImage(*odd_frame->GetImage())),
true);
446 tr1::shared_ptr<Frame> even_frame;
448 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
450 frame->AddImage(tr1::shared_ptr<QImage>(
new QImage(*even_frame->GetImage())),
false);
454 int samples_copied = 0;
459 int remaining_samples = mapped.
Samples.
total - samples_copied;
460 int number_to_copy = 0;
463 for (
int channel = 0; channel < channels_in_frame; channel++)
466 tr1::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
467 int original_samples = original_frame->GetAudioSamplesCount();
473 if (number_to_copy > remaining_samples)
474 number_to_copy = remaining_samples;
477 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.
Samples.
sample_start, number_to_copy, 1.0);
482 number_to_copy = original_samples;
483 if (number_to_copy > remaining_samples)
484 number_to_copy = remaining_samples;
487 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
493 if (number_to_copy > remaining_samples)
494 number_to_copy = remaining_samples;
497 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
502 samples_copied += number_to_copy;
515 final_cache.
Add(frame->number, frame);
521 return final_cache.
GetFrame(requested_frame);
532 float difference = target.
ToInt() - original.
ToInt();
534 int field_interval = 0;
535 int frame_interval = 0;
540 field_interval = round(fabs(original.
ToInt() / difference));
543 frame_interval = field_interval * 2.0f;
547 for (
float map = 1; map <=
frames.size(); map++)
550 cout <<
"Target frame #: " << map <<
" mapped to original frame #:\t(" << frame.
Odd.
Frame <<
" odd, " << frame.
Even.
Frame <<
" even)" << endl;
571 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Open",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
586 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
593 avresample_close(avr);
594 avresample_free(&avr);
613 root[
"type"] =
"FrameMapper";
625 bool success = reader.parse( value, root );
628 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
638 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
659 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ChangeMapping",
"target_fps.num", target_fps.
num,
"target_fps.den", target_fps.
num,
"target_pulldown", target_pulldown,
"target_sample_rate", target_sample_rate,
"target_channels", target_channels,
"target_channel_layout", target_channel_layout);
666 pulldown = target_pulldown;
679 avresample_close(avr);
680 avresample_free(&avr);
689 int total_frame_samples = 0;
690 int channels_in_frame = frame->GetAudioChannelsCount();
691 int sample_rate_in_frame = frame->SampleRate();
692 int samples_in_frame = frame->GetAudioSamplesCount();
693 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
695 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio",
"frame->number", frame->number,
"original_frame_number", original_frame_number,
"channels_in_frame", channels_in_frame,
"samples_in_frame", samples_in_frame,
"sample_rate_in_frame", sample_rate_in_frame,
"", -1);
698 float* frame_samples_float = NULL;
700 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
703 total_frame_samples = samples_in_frame * channels_in_frame;
706 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
709 for (
int s = 0; s < total_frame_samples; s++)
711 frame_samples[s] =
int(frame_samples_float[s] * (1 << 15));
715 delete[] frame_samples_float;
716 frame_samples_float = NULL;
718 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (got sample data from frame)",
"frame->number", frame->number,
"total_frame_samples", total_frame_samples,
"target channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"target sample_rate",
info.
sample_rate,
"samples_in_frame", samples_in_frame);
724 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
726 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
727 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
731 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio ERROR [" + (
string)
av_err2str(error_code) +
"]",
"error_code", error_code,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
732 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
738 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (adjust # of samples)",
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"sample_rate_in_frame", sample_rate_in_frame,
"info.channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"original_frame_number", original_frame_number);
743 audio_converted->nb_samples = total_frame_samples;
744 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
746 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (preparing for resample)",
"in_sample_fmt", AV_SAMPLE_FMT_S16,
"out_sample_fmt", AV_SAMPLE_FMT_S16,
"in_sample_rate", sample_rate_in_frame,
"out_sample_rate",
info.
sample_rate,
"in_channels", channels_in_frame,
"out_channels",
info.
channels);
755 avr = avresample_alloc_context();
756 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
758 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
759 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
760 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
762 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
764 avresample_open(avr);
768 nb_samples = avresample_convert(avr,
769 audio_converted->data,
770 audio_converted->linesize[0],
771 audio_converted->nb_samples,
773 audio_frame->linesize[0],
774 audio_frame->nb_samples);
778 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
781 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
784 av_freep(&audio_frame->data[0]);
786 av_freep(&audio_converted->data[0]);
788 frame_samples = NULL;
791 int channel_buffer_size = nb_samples;
794 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
"nb_samples", nb_samples,
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"channels_in_frame", channels_in_frame,
"info.channels",
info.
channels,
"info.channel_layout",
info.
channel_layout);
797 float *channel_buffer =
new float[channel_buffer_size];
800 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
803 for (
int z = 0; z < channel_buffer_size; z++)
804 channel_buffer[z] = 0.0f;
810 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
813 if (channel_filter == channel)
816 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
832 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
834 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Add audio to channel)",
"number of samples", position,
"channel_filter", channel_filter,
"", -1,
"", -1,
"", -1,
"", -1);
842 delete[] channel_buffer;
843 channel_buffer = NULL;
846 delete[] resampled_samples;
847 resampled_samples = NULL;
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
vector< MappedFrame > frames
void Add(long int frame_number, tr1::shared_ptr< Frame > frame)
Add a Frame to the cache.
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
tr1::shared_ptr< Frame > GetFrame(long int requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
This class represents a single frame of video (i.e. image & audio data)
void ResampleMappedAudio(tr1::shared_ptr< Frame > frame, long int original_frame_number)
Resample audio and map channels (if needed)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
~FrameMapper()
Destructor.
float duration
Length of time (in seconds)
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
This struct holds a single field (half a frame).
Exception when encoding audio packet.
void AddPoint(Point p)
Add a new point on the key-frame. Each point has a primary coordinate, a left handle, and a right handle.
This struct holds a the range of samples needed by this frame.
void SetMaxBytesFromInfo(long int number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
MappedFrame GetMappedFrame(long int TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
bool has_audio
Determines if this file has an audio stream.
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int height
The height of the video (in pixels)
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
bool IsOpen()
Determine if reader is open or closed.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
bool has_single_image
Determines if this file only contains a single image.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
#define av_err2str(errnum)
ReaderInfo info
Information about the current media file.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
This struct holds two fields which together make up a complete video frame.
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
string Json()
Get and Set JSON methods.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
int GetInt(long int index)
Get the rounded INT value at a specific index.
tr1::shared_ptr< Frame > GetFrame(long int frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
This namespace is the default namespace for all code in the openshot library.
virtual tr1::shared_ptr< Frame > GetFrame(long int number)=0
Do not apply pull-down techniques, just repeat or skip entire frames.
Linear curves are angular, straight lines between two points.
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
PulldownType
This enumeration determines how frame rates are increased or decreased.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
A Keyframe is a collection of Point instances, which is used to vary a number or property over time...
long int video_length
The number of frames in the video stream.
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
void SetJson(string value)
Load JSON string into this object.
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.
virtual bool IsOpen()=0
A thread safe version of GetFrame.