31 #include "../include/FFmpegReader.h" 36 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(
true), check_interlace(
false),
38 check_fps(
false), enable_seek(
true), rescaler_position(0), num_of_rescalers(
OPEN_MP_NUM_PROCESSORS), is_open(
false),
39 seek_audio_frame_found(0), seek_video_frame_found(0), prev_samples(0), prev_pts(0),
40 pts_total(0), pts_counter(0), is_duration_known(
false), largest_frame_processed(0),
41 current_video_frame(0), has_missing_frames(
false), num_packets_since_video_frame(0), num_checks_since_final(0) {
45 avcodec_register_all();
48 working_cache.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 30, info.width, info.height, info.sample_rate, info.channels);
49 missing_frames.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
50 final_cache.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
64 void FFmpegReader::InitScalers()
67 for (
int x = 0; x < num_of_rescalers; x++)
73 image_rescalers.push_back(img_convert_ctx);
81 if (abs(location.
frame - frame) >= 2)
85 int sample_diff = abs(location.
sample_start - sample_start);
86 if (location.
frame == frame && sample_diff >= 0 && sample_diff <= amount)
91 if (location.
frame > frame)
94 sample_diff = (samples_per_frame - sample_start) + location.
sample_start;
95 if (sample_diff >= 0 && sample_diff <= amount)
100 if (location.
frame < frame)
103 sample_diff = (samples_per_frame - location.
sample_start) + sample_start;
104 if (sample_diff >= 0 && sample_diff <= amount)
113 void FFmpegReader::RemoveScalers()
116 for (
int x = 0; x < num_of_rescalers; x++)
117 sws_freeContext(image_rescalers[x]);
120 image_rescalers.clear();
132 if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
133 throw InvalidFile(
"File could not be opened.", path);
136 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
137 throw NoStreamsFound(
"No streams found in file.", path);
142 for (
unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
145 if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
149 if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
153 if (videoStream == -1 && audioStream == -1)
154 throw NoStreamsFound(
"No video or audio streams found in this file.", path);
157 if (videoStream != -1)
163 pStream = pFormatCtx->streams[videoStream];
164 pCodecCtx = pFormatCtx->streams[videoStream]->codec;
170 AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
171 if (pCodec == NULL) {
172 throw InvalidCodec(
"A valid video codec could not be found for this file.", path);
175 if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
176 throw InvalidCodec(
"A video codec was found, but could not be opened.", path);
186 if (audioStream != -1)
192 aStream = pFormatCtx->streams[audioStream];
193 aCodecCtx = pFormatCtx->streams[audioStream]->codec;
199 AVCodec *aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
200 if (aCodec == NULL) {
201 throw InvalidCodec(
"A valid audio codec could not be found for this file.", path);
204 if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0)
205 throw InvalidCodec(
"An audio codec was found, but could not be opened.", path);
212 previous_packet_location.
frame = -1;
230 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
241 avcodec_flush_buffers(pCodecCtx);
242 avcodec_close(pCodecCtx);
246 avcodec_flush_buffers(aCodecCtx);
247 avcodec_close(aCodecCtx);
252 working_cache.
Clear();
253 missing_frames.
Clear();
258 processed_video_frames.clear();
259 processed_audio_frames.clear();
260 processing_video_frames.clear();
261 processing_audio_frames.clear();
262 missing_audio_frames.clear();
263 missing_video_frames.clear();
264 checked_frames.clear();
268 avformat_close_input(&pFormatCtx);
269 av_freep(&pFormatCtx);
273 largest_frame_processed = 0;
274 seek_audio_frame_found = 0;
275 seek_video_frame_found = 0;
276 current_video_frame = 0;
277 has_missing_frames =
false;
281 void FFmpegReader::UpdateAudioInfo()
285 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
288 if (aCodecCtx->channel_layout == 0)
289 aCodecCtx->channel_layout = av_get_default_channel_layout( aCodecCtx->channels );;
299 if (aStream->duration > 0.0f && aStream->duration >
info.
duration)
325 void FFmpegReader::UpdateVideoInfo()
329 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
341 if (pStream->sample_aspect_ratio.num != 0)
346 else if (pCodecCtx->sample_aspect_ratio.num != 0)
392 is_duration_known =
false;
397 is_duration_known =
true;
423 throw ReaderClosed(
"The FFmpegReader is closed. Call Open() before calling this method.", path);
426 if (requested_frame < 1)
432 throw InvalidFile(
"Could not detect the duration of the video or audio stream.", path);
435 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"requested_frame", requested_frame,
"last_frame", last_frame,
"", -1,
"", -1,
"", -1,
"", -1);
441 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
452 if (has_missing_frames)
453 CheckMissingFrame(requested_frame);
457 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame on 2nd look", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
468 if (last_frame == 0 && requested_frame != 1)
473 long int diff = requested_frame - last_frame;
474 if (diff >= 1 && diff <= 20)
477 return ReadStream(requested_frame);
484 Seek(requested_frame);
494 return ReadStream(requested_frame);
501 tr1::shared_ptr<Frame> FFmpegReader::ReadStream(
long int requested_frame)
504 bool end_of_stream =
false;
505 bool check_seek =
false;
506 bool frame_finished =
false;
507 int packet_error = -1;
510 int packets_processed = 0;
516 omp_set_nested(
true);
519 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream",
"requested_frame", requested_frame,
"OPEN_MP_NUM_PROCESSORS",
OPEN_MP_NUM_PROCESSORS,
"", -1,
"", -1,
"", -1,
"", -1);
529 packet_error = GetNextPacket();
532 while (processing_video_frames.size() + processing_audio_frames.size() >= minimum_packets)
536 if (packet_error < 0)
539 end_of_stream =
true;
544 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (GetNextPacket)",
"requested_frame", requested_frame,
"processing_video_frames.size()", processing_video_frames.size(),
"processing_audio_frames.size()", processing_audio_frames.size(),
"minimum_packets", minimum_packets,
"packets_processed", packets_processed,
"", -1);
550 num_packets_since_video_frame = 0;
554 #pragma omp critical (openshot_seek) 555 check_seek = CheckSeek(
true);
561 RemoveAVPacket(packet);
568 frame_finished = GetAVFrame();
574 UpdatePTSOffset(
true);
577 ProcessVideoPacket(requested_frame);
582 else if (
info.
has_audio && packet->stream_index == audioStream)
585 num_packets_since_video_frame++;
589 #pragma omp critical (openshot_seek) 590 check_seek = CheckSeek(
false);
596 RemoveAVPacket(packet);
603 UpdatePTSOffset(
false);
613 bool is_cache_found =
false;
616 CheckMissingFrame(requested_frame);
619 CheckWorkingFrames(
false, requested_frame);
629 if ((is_cache_found && packets_processed >= minimum_packets))
638 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (Completed)",
"packets_processed", packets_processed,
"end_of_stream", end_of_stream,
"largest_frame_processed", largest_frame_processed,
"Working Cache Count", working_cache.
Count(),
"", -1,
"", -1);
643 CheckWorkingFrames(end_of_stream, requested_frame);
660 tr1::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
669 int FFmpegReader::GetNextPacket()
671 int found_packet = 0;
672 AVPacket *next_packet =
new AVPacket();
673 found_packet = av_read_frame(pFormatCtx, next_packet);
675 if (found_packet >= 0)
677 #pragma omp critical (packet_cache) 680 packets[next_packet] = next_packet;
683 packet = packets[next_packet];
690 av_free_packet(next_packet);
699 bool FFmpegReader::GetAVFrame()
701 int frameFinished = -1;
705 #pragma omp critical (packet_cache) 706 avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
713 AVPicture *copyFrame =
new AVPicture();
715 av_picture_copy(copyFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt,
info.
width,
info.
height);
717 #pragma omp critical (packet_cache) 720 frames[copyFrame] = copyFrame;
721 pFrame = frames[copyFrame];
725 if (!check_interlace)
727 check_interlace =
true;
736 RemoveAVPacket(packet);
743 return frameFinished;
747 bool FFmpegReader::CheckSeek(
bool is_video)
754 if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
762 long int max_seeked_frame = seek_audio_frame_found;
763 if (seek_video_frame_found > max_seeked_frame)
764 max_seeked_frame = seek_video_frame_found;
767 if (max_seeked_frame >= seeking_frame)
770 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Too far, seek again)",
"is_video_seek", is_video_seek,
"max_seeked_frame", max_seeked_frame,
"seeking_frame", seeking_frame,
"seeking_pts", seeking_pts,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
773 Seek(seeking_frame - (20 * seek_count * seek_count));
778 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Successful)",
"is_video_seek", is_video_seek,
"current_pts", packet->pts,
"seeking_pts", seeking_pts,
"seeking_frame", seeking_frame,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
792 void FFmpegReader::ProcessVideoPacket(
long int requested_frame)
795 long int current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
798 if (!seek_video_frame_found && is_seeking)
799 seek_video_frame_found = current_frame;
802 if ((!has_missing_frames and current_frame < (requested_frame - 20)) or (current_frame == -1))
805 RemoveAVFrame(pFrame);
806 RemoveAVPacket(packet);
809 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Skipped)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
816 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Before)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
823 AVPacket *my_packet = packets[packet];
824 AVPicture *my_frame = frames[pFrame];
827 SwsContext *img_convert_ctx = image_rescalers[rescaler_position];
829 if (rescaler_position == num_of_rescalers)
830 rescaler_position = 0;
834 processing_video_frames[current_frame] = current_frame;
836 #pragma omp task firstprivate(current_frame, my_packet, my_frame, height, width, video_length, pix_fmt, img_convert_ctx) 839 AVFrame *pFrameRGB = NULL;
841 uint8_t *buffer = NULL;
845 if (pFrameRGB == NULL)
846 throw OutOfBoundsFrame(
"Convert Image Broke!", current_frame, video_length);
849 numBytes = avpicture_get_size(
PIX_FMT_RGBA, width, height);
850 buffer = (uint8_t *) av_malloc(numBytes *
sizeof(uint8_t));
855 avpicture_fill((AVPicture *) pFrameRGB, buffer,
PIX_FMT_RGBA, width, height);
858 sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
859 height, pFrameRGB->data, pFrameRGB->linesize);
862 tr1::shared_ptr<Frame> f = CreateFrame(current_frame);
865 f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
868 working_cache.
Add(f->number, f);
871 last_video_frame = f;
878 RemoveAVFrame(my_frame);
879 RemoveAVPacket(my_packet);
884 processing_video_frames.erase(current_frame);
885 processed_video_frames[current_frame] = current_frame;
889 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (After)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"f->number", f->number,
"", -1,
"", -1,
"", -1);
896 void FFmpegReader::ProcessAudioPacket(
long int requested_frame,
long int target_frame,
int starting_sample)
899 if (!seek_audio_frame_found && is_seeking)
900 seek_audio_frame_found = target_frame;
903 if (!has_missing_frames and target_frame < (requested_frame - 20))
906 RemoveAVPacket(packet);
909 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Skipped)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
916 AVPacket *my_packet = packets[packet];
919 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Before)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
922 int frame_finished = 0;
926 int packet_samples = 0;
931 int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, my_packet);
933 if (frame_finished) {
936 int planar = av_sample_fmt_is_planar(aCodecCtx->sample_fmt);
938 data_size = av_samples_get_buffer_size(&plane_size,
940 audio_frame->nb_samples,
941 aCodecCtx->sample_fmt, 1);
944 packet_samples = audio_frame->nb_samples * aCodecCtx->channels;
948 int pts_remaining_samples = packet_samples /
info.
channels;
951 int adjusted_pts = packet->pts + audio_pts_offset;
956 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info A)",
"pts_counter", pts_counter,
"PTS", adjusted_pts,
"Offset", audio_pts_offset,
"PTS Diff", adjusted_pts - prev_pts,
"Samples", pts_remaining_samples,
"Sample PTS ratio",
float(adjusted_pts - prev_pts) / pts_remaining_samples);
957 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info B)",
"Sample Diff", pts_remaining_samples - prev_samples - prev_pts,
"Total", pts_total,
"PTS Seconds", audio_seconds,
"Sample Seconds", sample_seconds,
"Seconds Diff", audio_seconds - sample_seconds,
"raw samples", packet_samples);
960 prev_pts = adjusted_pts;
961 pts_total += pts_remaining_samples;
963 prev_samples = pts_remaining_samples;
968 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
971 while (pts_remaining_samples)
977 int samples = samples_per_frame - previous_packet_location.
sample_start;
978 if (samples > pts_remaining_samples)
979 samples = pts_remaining_samples;
982 pts_remaining_samples -= samples;
984 if (pts_remaining_samples > 0) {
986 previous_packet_location.
frame++;
992 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
1005 #pragma omp task firstprivate(requested_frame, target_frame, starting_sample, my_packet, audio_frame) 1010 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (ReSample)",
"packet_samples", packet_samples,
"info.channels",
info.
channels,
"info.sample_rate",
info.
sample_rate,
"aCodecCtx->sample_fmt", aCodecCtx->sample_fmt,
"AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16,
"", -1);
1015 audio_converted->nb_samples = audio_frame->nb_samples;
1016 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1018 AVAudioResampleContext *avr = NULL;
1023 avr = avresample_alloc_context();
1024 av_opt_set_int(avr,
"in_channel_layout", aCodecCtx->channel_layout, 0);
1025 av_opt_set_int(avr,
"out_channel_layout", aCodecCtx->channel_layout, 0);
1026 av_opt_set_int(avr,
"in_sample_fmt", aCodecCtx->sample_fmt, 0);
1027 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1032 int r = avresample_open(avr);
1035 nb_samples = avresample_convert(avr,
1036 audio_converted->data,
1037 audio_converted->linesize[0],
1038 audio_converted->nb_samples,
1040 audio_frame->linesize[0],
1041 audio_frame->nb_samples);
1045 memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels);
1048 avresample_close(avr);
1049 avresample_free(&avr);
1053 av_free(audio_converted->data[0]);
1056 int starting_frame_number = -1;
1057 bool partial_frame =
true;
1058 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
1061 starting_frame_number = target_frame;
1062 int channel_buffer_size = packet_samples /
info.
channels;
1063 float *channel_buffer =
new float[channel_buffer_size];
1066 for (
int z = 0; z < channel_buffer_size; z++)
1067 channel_buffer[z] = 0.0f;
1073 for (
int sample = 0; sample < packet_samples; sample++)
1076 if (channel_filter == channel)
1079 channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1095 int start = starting_sample;
1096 int remaining_samples = channel_buffer_size;
1097 float *iterate_channel_buffer = channel_buffer;
1098 while (remaining_samples > 0)
1104 int samples = samples_per_frame - start;
1105 if (samples > remaining_samples)
1106 samples = remaining_samples;
1109 tr1::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1112 if (samples_per_frame == start + samples)
1113 partial_frame =
false;
1115 partial_frame =
true;
1119 f->AddAudio(
true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1122 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (f->AddAudio)",
"frame", starting_frame_number,
"start", start,
"samples", samples,
"channel", channel_filter,
"partial_frame", partial_frame,
"samples_per_frame", samples_per_frame);
1125 working_cache.
Add(f->number, f);
1128 remaining_samples -= samples;
1131 if (remaining_samples > 0)
1132 iterate_channel_buffer += samples;
1135 starting_frame_number++;
1142 delete[] channel_buffer;
1143 channel_buffer = NULL;
1144 iterate_channel_buffer = NULL;
1155 for (
long int f = target_frame; f < starting_frame_number; f++) {
1159 processing_audio_frames.erase(processing_audio_frames.find(f));
1162 if (processing_audio_frames.count(f) == 0)
1164 processed_audio_frames[f] = f;
1167 if (target_frame == starting_frame_number) {
1169 processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1174 RemoveAVPacket(my_packet);
1177 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (After)",
"requested_frame", requested_frame,
"starting_frame", target_frame,
"end_frame", starting_frame_number - 1,
"", -1,
"", -1,
"", -1);
1184 #pragma omp taskwait 1193 void FFmpegReader::Seek(
long int requested_frame)
throw(
TooManySeeks)
1196 if (requested_frame < 1)
1197 requested_frame = 1;
1202 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Seek",
"requested_frame", requested_frame,
"seek_count", seek_count,
"last_frame", last_frame,
"", -1,
"", -1,
"", -1);
1205 working_cache.
Clear();
1206 missing_frames.
Clear();
1211 processing_audio_frames.clear();
1212 processing_video_frames.clear();
1213 processed_video_frames.clear();
1214 processed_audio_frames.clear();
1215 duplicate_video_frames.clear();
1216 missing_audio_frames.clear();
1217 missing_video_frames.clear();
1218 checked_frames.clear();
1223 current_video_frame = 0;
1224 largest_frame_processed = 0;
1225 num_checks_since_final = 0;
1226 num_packets_since_video_frame = 0;
1227 has_missing_frames =
false;
1233 int buffer_amount = 6;
1234 if (requested_frame - buffer_amount < 20)
1242 if (seek_count == 1) {
1245 seeking_pts = ConvertFrameToVideoPTS(1);
1247 seek_audio_frame_found = 0;
1248 seek_video_frame_found = 0;
1253 bool seek_worked =
false;
1254 int64_t seek_target = 0;
1259 seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1261 fprintf(stderr,
"%s: error while seeking video stream\n", pFormatCtx->filename);
1265 is_video_seek =
true;
1273 seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1275 fprintf(stderr,
"%s: error while seeking audio stream\n", pFormatCtx->filename);
1279 is_video_seek =
false;
1289 avcodec_flush_buffers(aCodecCtx);
1293 avcodec_flush_buffers(pCodecCtx);
1296 previous_packet_location.
frame = -1;
1301 if (seek_count == 1) {
1303 seeking_pts = seek_target;
1304 seeking_frame = requested_frame;
1306 seek_audio_frame_found = 0;
1307 seek_video_frame_found = 0;
1329 long int FFmpegReader::GetVideoPTS()
1331 int current_pts = 0;
1332 if(packet->dts != AV_NOPTS_VALUE)
1333 current_pts = packet->dts;
1340 void FFmpegReader::UpdatePTSOffset(
bool is_video)
1346 if (video_pts_offset == 99999)
1348 video_pts_offset = 0 - GetVideoPTS();
1353 if (audio_pts_offset == 99999)
1355 audio_pts_offset = 0 - packet->pts;
1360 long int FFmpegReader::ConvertVideoPTStoFrame(
long int pts)
1363 pts = pts + video_pts_offset;
1364 long int previous_video_frame = current_video_frame;
1373 if (current_video_frame == 0)
1374 current_video_frame = frame;
1378 if (frame == previous_video_frame) {
1379 duplicate_video_frames.insert(pair<int,int>(frame, frame));
1386 current_video_frame++;
1388 if (current_video_frame < frame)
1390 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)",
"calculated frame", frame,
"previous_video_frame", previous_video_frame,
"current_video_frame", current_video_frame,
"", -1,
"", -1,
"", -1);
1395 while (current_video_frame < frame) {
1396 if (!missing_video_frames.count(current_video_frame)) {
1397 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)",
"current_video_frame", current_video_frame,
"previous_video_frame", previous_video_frame,
"", -1,
"", -1,
"", -1,
"", -1);
1398 missing_video_frames.insert(pair<long int, long int>(previous_video_frame, current_video_frame));
1402 has_missing_frames =
true;
1405 current_video_frame++;
1414 long int FFmpegReader::ConvertFrameToVideoPTS(
long int frame_number)
1423 return video_pts - video_pts_offset;
1427 long int FFmpegReader::ConvertFrameToAudioPTS(
long int frame_number)
1436 return audio_pts - audio_pts_offset;
1440 AudioLocation FFmpegReader::GetAudioPTSLocation(
long int pts)
1443 pts = pts + audio_pts_offset;
1452 int whole_frame = int(frame);
1455 double sample_start_percentage = frame - double(whole_frame);
1461 int sample_start = round(
double(samples_per_frame) * sample_start_percentage);
1464 if (whole_frame < 1)
1466 if (sample_start < 0)
1473 if (previous_packet_location.
frame != -1 && location.
is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1475 int orig_frame = location.
frame;
1479 if (previous_packet_location.
sample_start <= samples_per_frame)
1482 location.
frame = previous_packet_location.
frame;
1492 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)",
"Source Frame", orig_frame,
"Source Audio Sample", orig_start,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1);
1496 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)",
"Previous location frame", previous_packet_location.
frame,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1,
"", -1);
1499 for (
long int audio_frame = previous_packet_location.
frame; audio_frame < location.
frame; audio_frame++) {
1500 if (!missing_audio_frames.count(audio_frame)) {
1501 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (tracking missing frame)",
"missing_audio_frame", audio_frame,
"previous_audio_frame", previous_packet_location.
frame,
"new location frame", location.
frame,
"", -1,
"", -1,
"", -1);
1502 missing_audio_frames.insert(pair<long int, long int>(previous_packet_location.
frame - 1, audio_frame));
1508 previous_packet_location = location;
1515 tr1::shared_ptr<Frame> FFmpegReader::CreateFrame(
long int requested_frame)
1518 tr1::shared_ptr<Frame> output = working_cache.
GetFrame(requested_frame);
1527 working_cache.
Add(requested_frame, output);
1530 if (requested_frame > largest_frame_processed)
1531 largest_frame_processed = requested_frame;
1539 bool FFmpegReader::IsPartialFrame(
long int requested_frame) {
1542 bool seek_trash =
false;
1543 long int max_seeked_frame = seek_audio_frame_found;
1544 if (seek_video_frame_found > max_seeked_frame)
1545 max_seeked_frame = seek_video_frame_found;
1546 if ((
info.
has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1547 (
info.
has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1554 bool FFmpegReader::CheckMissingFrame(
long int requested_frame)
1560 int checked_count = 0;
1563 if (checked_frames.count(requested_frame) == 0)
1564 checked_frames[requested_frame] = 1;
1566 checked_frames[requested_frame]++;
1567 checked_count = checked_frames[requested_frame];
1570 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame",
"requested_frame", requested_frame,
"has_missing_frames", has_missing_frames,
"missing_video_frames.size()", missing_video_frames.size(),
"checked_count", checked_count,
"", -1,
"", -1);
1574 map<long int, long int>::iterator itr;
1575 bool found_missing_frame =
false;
1577 for (itr = missing_video_frames.begin(); itr != missing_video_frames.end(); ++itr) {
1579 if (requested_frame == itr->second) {
1581 found_missing_frame =
true;
1584 tr1::shared_ptr<Frame> parent_frame = missing_frames.
GetFrame(itr->first);
1587 tr1::shared_ptr<Frame> missing_frame = CreateFrame(itr->second);
1590 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"previous_frame->number", itr->first,
"", -1,
"", -1,
"", -1);
1593 if (parent_frame != NULL) {
1595 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"previous_frame->number", parent_frame->number,
"", -1,
"", -1,
"", -1);
1598 missing_frame->AddImage(tr1::shared_ptr<QImage>(
new QImage(*parent_frame->GetImage())));
1600 processed_video_frames[missing_frame->number] = missing_frame->number;
1601 processed_audio_frames[missing_frame->number] = missing_frame->number;
1607 working_cache.
Remove(missing_frame->number);
1610 last_frame = missing_frame->number;
1616 if (found_missing_frame) {
1618 missing_video_frames.erase(itr);
1622 found_missing_frame =
false;
1623 for (itr = missing_audio_frames.begin(); itr != missing_audio_frames.end(); ++itr) {
1625 if (requested_frame == itr->second) {
1627 found_missing_frame =
true;
1630 tr1::shared_ptr<Frame> parent_frame = missing_frames.
GetFrame(itr->first);
1633 tr1::shared_ptr<Frame> missing_frame = CreateFrame(itr->second);
1636 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (Is Previous Audio Frame Final)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"previous_frame->number", itr->first,
"", -1,
"", -1,
"", -1);
1639 if (parent_frame != NULL) {
1641 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (AddImage from Previous Audio Frame)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"previous_frame->number", parent_frame->number,
"", -1,
"", -1,
"", -1);
1644 missing_frame->AddImage(tr1::shared_ptr<QImage>(
new QImage(*parent_frame->GetImage())));
1646 processed_video_frames[missing_frame->number] = missing_frame->number;
1647 processed_audio_frames[missing_frame->number] = missing_frame->number;
1653 working_cache.
Remove(missing_frame->number);
1656 last_frame = missing_frame->number;
1662 if (found_missing_frame) {
1664 missing_audio_frames.erase(itr);
1668 return found_missing_frame;
1672 void FFmpegReader::CheckWorkingFrames(
bool end_of_stream,
long int requested_frame)
1686 CheckMissingFrame(f->number);
1689 int checked_count = 0;
1691 bool is_video_ready =
false;
1692 bool is_audio_ready =
false;
1695 is_video_ready = processed_video_frames.count(f->number);
1696 is_audio_ready = processed_audio_frames.count(f->number);
1699 checked_count = checked_frames[f->number];
1702 if (previous_packet_location.
frame == f->number && !end_of_stream)
1703 is_audio_ready =
false;
1704 bool is_seek_trash = IsPartialFrame(f->number);
1711 if (checked_count > 40 && (!is_video_ready || !is_audio_ready)) {
1713 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (exceeded checked_count)",
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames.size()", checked_frames.size(),
"", -1);
1715 if (
info.
has_video && !is_video_ready && last_video_frame) {
1717 f->AddImage(tr1::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1718 is_video_ready =
true;
1724 is_audio_ready =
true;
1729 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames",
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames.size()", checked_frames.size(),
"", -1);
1732 if ((!end_of_stream && is_video_ready && is_audio_ready && f->number <= requested_frame) || end_of_stream || is_seek_trash)
1735 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (mark frame as final)",
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1,
"", -1);
1740 num_checks_since_final = 0;
1748 if (!missing_video_frames.count(current_video_frame)) {
1750 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (add frame to missing cache)",
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Missing Cache Count", missing_frames.
Count(),
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1);
1752 missing_frames.
Add(f->number, f);
1757 working_cache.
Remove(f->number);
1760 last_frame = f->number;
1763 checked_frames.erase(f->number);
1767 working_cache.
Remove(f->number);
1777 void FFmpegReader::CheckFPS()
1782 int first_second_counter = 0;
1783 int second_second_counter = 0;
1784 int third_second_counter = 0;
1785 int forth_second_counter = 0;
1786 int fifth_second_counter = 0;
1789 int threshold = 500;
1795 if (GetNextPacket() < 0)
1800 if (packet->stream_index == videoStream)
1806 UpdatePTSOffset(
true);
1809 long int pts = GetVideoPTS();
1812 RemoveAVFrame(pFrame);
1815 RemoveAVPacket(packet);
1818 pts += video_pts_offset;
1824 if (video_seconds <= 1.0)
1825 first_second_counter++;
1826 else if (video_seconds > 1.0 && video_seconds <= 2.0)
1827 second_second_counter++;
1828 else if (video_seconds > 2.0 && video_seconds <= 3.0)
1829 third_second_counter++;
1830 else if (video_seconds > 3.0 && video_seconds <= 4.0)
1831 forth_second_counter++;
1832 else if (video_seconds > 4.0 && video_seconds <= 5.0)
1833 fifth_second_counter++;
1840 RemoveAVPacket(packet);
1844 RemoveAVPacket(packet);
1850 if (iterations > threshold)
1855 if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0)
1864 int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1865 int avg_fps = round(sum_fps / 4.0f);
1873 double diff = fps - double(avg_fps);
1876 if (diff <= -1 || diff >= 1)
1880 diff = half_fps - double(avg_fps);
1883 if (diff <= -1 || diff >= 1)
1900 void FFmpegReader::RemoveAVFrame(AVPicture* remove_frame)
1902 #pragma omp critical (packet_cache) 1905 if (frames.count(remove_frame))
1908 avpicture_free(frames[remove_frame]);
1911 frames.erase(remove_frame);
1914 delete remove_frame;
1921 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1923 #pragma omp critical (packet_cache) 1926 if (packets.count(remove_packet))
1929 av_free_packet(remove_packet);
1932 packets.erase(remove_packet);
1935 delete remove_packet;
1942 long int FFmpegReader::GetSmallestVideoFrame()
1945 map<long int, long int>::iterator itr;
1946 int smallest_frame = -1;
1947 for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1949 if (itr->first < smallest_frame || smallest_frame == -1)
1950 smallest_frame = itr->first;
1954 return smallest_frame;
1958 long int FFmpegReader::GetSmallestAudioFrame()
1961 map<long int, long int>::iterator itr;
1962 int smallest_frame = -1;
1964 for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
1966 if (itr->first < smallest_frame || smallest_frame == -1)
1967 smallest_frame = itr->first;
1971 return smallest_frame;
1986 root[
"type"] =
"FFmpegReader";
1987 root[
"path"] = path;
1998 Json::Reader reader;
1999 bool success = reader.parse( value, root );
2002 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
2012 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
2023 if (!root[
"path"].isNull())
2024 path = root[
"path"].asString();
#define AV_RESET_FRAME(av_frame)
long long file_size
Size of file (in bytes)
tr1::shared_ptr< Frame > GetFrame(long int requested_frame)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
void Add(long int frame_number, tr1::shared_ptr< Frame > frame)
Add a Frame to the cache.
CriticalSection processingCriticalSection
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
void Remove(long int frame_number)
Remove a specific frame.
This class represents a single frame of video (i.e. image & audio data)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
float duration
Length of time (in seconds)
string acodec
The name of the audio codec used to encode / decode the video stream.
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
FFmpegReader(string path)
void SetMaxBytesFromInfo(long int number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
int audio_bit_rate
The bit rate of the audio stream (in bytes)
bool has_audio
Determines if this file has an audio stream.
Exception when no valid codec is found for a file.
int audio_stream_index
The index of the audio stream.
Exception when no streams are found in the file.
int height
The height of the video (in pixels)
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
Cache final_cache
Final cache object used to hold final frames.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
tr1::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
ReaderInfo info
Information about the current media file.
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Exception for frames that are out of bounds.
int is_near(AudioLocation location, int samples_per_frame, int amount)
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
tr1::shared_ptr< Frame > GetFrame(long int frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
This struct holds the associated video frame and starting sample # for an audio packet.
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
string vcodec
The name of the video codec used to encode / decode the video stream.
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
long int video_length
The number of frames in the video stream.
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
long int Count()
Count the frames in the queue.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.