From 9b713a489d2696a7578272823d7b53a2ed1ed3a2 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Wed, 6 Mar 2019 14:50:36 -0500 Subject: [PATCH] fix sleep time by using a comparison between the frame display time and the distance between now and when we started playing. handle infinite fps --- src/zm_eventstream.cpp | 48 +++++++++++++++++++++++++---------------- src/zm_ffmpeg_input.cpp | 40 ++++++++++++++++++---------------- src/zm_stream.cpp | 2 +- src/zm_videostore.cpp | 9 -------- 4 files changed, 51 insertions(+), 48 deletions(-) diff --git a/src/zm_eventstream.cpp b/src/zm_eventstream.cpp index 03996e49c..d46c9b385 100644 --- a/src/zm_eventstream.cpp +++ b/src/zm_eventstream.cpp @@ -228,7 +228,7 @@ bool EventStream::loadEventData(uint64_t event_id) { //timestamp = atof(dbrow[1]); double delta = atof(dbrow[2]); int id_diff = id - last_id; - double frame_delta = id_diff ? (delta-last_delta)/id_diff : (delta - last_delta); + double frame_delta = id_diff ? (delta-last_delta)/id_diff : (delta-last_delta); // Fill in data between bulk frames if ( id_diff > 1 ) { for ( int i = last_id+1; i < id; i++ ) { @@ -262,8 +262,8 @@ bool EventStream::loadEventData(uint64_t event_id) { ); } if ( mysql_errno( &dbconn ) ) { - Error( "Can't fetch row: %s", mysql_error( &dbconn ) ); - exit( mysql_errno( &dbconn ) ); + Error("Can't fetch row: %s", mysql_error(&dbconn)); + exit(mysql_errno(&dbconn)); } mysql_free_result(result); @@ -676,10 +676,6 @@ Debug(1, "Loading image"); // Get the frame from the mp4 input Debug(1,"Getting frame from ffmpeg"); AVFrame *frame; - if ( curr_frame_id == 1 ) { - // Special case, first frame, we want to send the initial keyframe. - frame = ffmpeg_input->get_frame( ffmpeg_input->get_video_stream_id(), 0 ); - } FrameData *frame_data = &event_data->frames[curr_frame_id-1]; frame = ffmpeg_input->get_frame( ffmpeg_input->get_video_stream_id(), frame_data->offset ); if ( frame ) { @@ -790,7 +786,9 @@ void EventStream::runStream() { Debug(3, "frame rate is: (%f)", (double)event_data->frame_count/event_data->duration); updateFrameRate((double)event_data->frame_count/event_data->duration); - gettimeofday(&start, NULL); + gettimeofday(&start, NULL); + uint64_t start_usec = start.tv_sec * 1000000 + start.tv_usec; + uint64_t last_frame_offset = 0; while ( !zm_terminate ) { gettimeofday(&now, NULL); @@ -868,7 +866,7 @@ Debug(3,"cur_frame_id (%d-1) mod frame_mod(%d)",curr_frame_id, frame_mod); Debug(3,"delta %u = base_fps(%f)/effective fps(%f)", delta_us, base_fps, effective_fps); // but must not exceed maxfps delta_us = max(delta_us, 1000000 / maxfps); - Debug(3,"delta %u = base_fps(%f)/effective fps(%f)", delta_us, base_fps, effective_fps); + Debug(3,"delta %u = base_fps(%f)/effective fps(%f) from 30fps", delta_us, base_fps, effective_fps); send_frame = true; } } else if ( step != 0 ) { @@ -896,13 +894,11 @@ Debug(3,"cur_frame_id (%d-1) mod frame_mod(%d)",curr_frame_id, frame_mod); //Debug(3,"Not sending frame"); } - curr_stream_time = frame_data->timestamp; if ( !paused ) { - curr_frame_id += (replay_rate>0) ? 1 : -1; - - + // +/- 1? What if we are skipping frames? + curr_frame_id += (replay_rate>0) ? frame_mod : -1*frame_mod; if ( (mode == MODE_SINGLE) && ((unsigned int)curr_frame_id == event_data->frame_count) ) { Debug(2, "Have mode==MODE_SINGLE and at end of event, looping back to start"); @@ -910,14 +906,28 @@ Debug(3,"cur_frame_id (%d-1) mod frame_mod(%d)",curr_frame_id, frame_mod); } frame_data = &event_data->frames[curr_frame_id-1]; + // sending the frame may have taken some time, so reload now + gettimeofday(&now, NULL); uint64_t now_usec = (now.tv_sec * 1000000 + now.tv_usec); - uint64_t start_usec = (start.tv_sec * 1000000 + start.tv_usec); - uint64_t frame_delta = frame_data->delta*1000000; + // frame_data->delta is the time since last frame as a float in seconds + // but what if we are skipping frames? We need the distance from the last frame sent + // Also, what about reverse? needs to be absolute value - delta_us = frame_delta - (now_usec - start_usec); - Debug(2, "New delta_us now %" PRIu64 " - start %" PRIu64 " = %d - frame %" PRIu64 " = %d", - now_usec, start_usec, now_usec-start_usec, frame_delta, - delta_us); + // There are two ways to go about this, not sure which is correct. + // you can calculate the relationship between now and the start + // or calc the relationship from the last frame. I think from the start is better as it self-corrects + + if ( last_frame_offset ) { + // We assume that we are going forward and the next frame is in the future. + delta_us = frame_data->offset * 1000000 - (now_usec-start_usec); + // - (now_usec - start_usec); + Debug(2, "New delta_us now %" PRIu64 " - start %" PRIu64 " = %d offset %" PRId64 " - elapsed = %dusec", + now_usec, start_usec, now_usec-start_usec, frame_data->offset * 1000000, delta_us); + } else { + Debug(2, "No last frame_offset, no sleep"); + delta_us = 0; + } + last_frame_offset = frame_data->offset * 1000000; if ( send_frame && type != STREAM_MPEG ) { if ( delta_us > 0 ) { diff --git a/src/zm_ffmpeg_input.cpp b/src/zm_ffmpeg_input.cpp index 9f0336bbf..d9c7c2fd3 100644 --- a/src/zm_ffmpeg_input.cpp +++ b/src/zm_ffmpeg_input.cpp @@ -110,7 +110,7 @@ AVFrame *FFmpeg_Input::get_frame( int stream_id ) { char errbuf[AV_ERROR_MAX_STRING_SIZE]; while ( !frameComplete ) { - int ret = av_read_frame( input_format_context, &packet ); + int ret = av_read_frame(input_format_context, &packet); if ( ret < 0 ) { av_strerror(ret, errbuf, AV_ERROR_MAX_STRING_SIZE); if ( @@ -135,9 +135,10 @@ AVFrame *FFmpeg_Input::get_frame( int stream_id ) { #if LIBAVCODEC_VERSION_CHECK(57, 64, 0, 64, 0) ret = avcodec_send_packet(context, &packet); if ( ret < 0 ) { - av_strerror( ret, errbuf, AV_ERROR_MAX_STRING_SIZE ); - Error( "Unable to send packet at frame %d: %s, continuing", streams[packet.stream_index].frame_count, errbuf ); - zm_av_packet_unref( &packet ); + av_strerror(ret, errbuf, AV_ERROR_MAX_STRING_SIZE); + Error("Unable to send packet at frame %d: %s, continuing", + streams[packet.stream_index].frame_count, errbuf); + zm_av_packet_unref(&packet); continue; } @@ -159,12 +160,12 @@ AVFrame *FFmpeg_Input::get_frame( int stream_id ) { } } else { #endif - if ( frame ) { - av_frame_free(&frame); - frame = zm_av_frame_alloc(); - } else { - frame = zm_av_frame_alloc(); - } + if ( frame ) { + av_frame_free(&frame); + frame = zm_av_frame_alloc(); + } else { + frame = zm_av_frame_alloc(); + } //Debug(1,"Getting frame %d", streams[packet.stream_index].frame_count); ret = avcodec_receive_frame(context, frame); if ( ret < 0 ) { @@ -181,12 +182,12 @@ AVFrame *FFmpeg_Input::get_frame( int stream_id ) { frameComplete = 1; # else - if ( frame ) { - av_frame_free(&frame); - frame = zm_av_frame_alloc(); - } else { - frame = zm_av_frame_alloc(); - } + if ( frame ) { + av_frame_free(&frame); + frame = zm_av_frame_alloc(); + } else { + frame = zm_av_frame_alloc(); + } ret = zm_avcodec_decode_video(context, frame, &frameComplete, &packet); if ( ret < 0 ) { av_strerror(ret, errbuf, AV_ERROR_MAX_STRING_SIZE); @@ -198,7 +199,7 @@ AVFrame *FFmpeg_Input::get_frame( int stream_id ) { #endif } // end if it's the right stream - zm_av_packet_unref( &packet ); + zm_av_packet_unref(&packet); } // end while ! frameComplete return frame; @@ -220,7 +221,7 @@ AVFrame *FFmpeg_Input::get_frame( int stream_id, double at ) { return frame; } if ( frame->pts < seek_target ) { - Debug(2, "Frame pts %" PRId64 " pkt_pts %" PRId64 " duration %" PRId64, frame->pts, frame->pkt_pts, frame->pkt_duration); + Debug(2, "Frame pts %" PRId64 " duration %" PRId64, frame->pts, frame->pkt_duration); while ( frame && (frame->pts < seek_target) ) { if ( ! get_frame(stream_id) ) return frame; @@ -237,9 +238,10 @@ AVFrame *FFmpeg_Input::get_frame( int stream_id, double at ) { } } else { + // No previous frame... are we asking for first frame? // Must go for a keyframe if ( ( ret = av_seek_frame(input_format_context, stream_id, seek_target, - AVSEEK_FLAG_FRAME + AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_FRAME ) < 0 ) ) { Error("Unable to seek in stream"); return NULL; diff --git a/src/zm_stream.cpp b/src/zm_stream.cpp index b34773ea5..20d4f1632 100644 --- a/src/zm_stream.cpp +++ b/src/zm_stream.cpp @@ -62,7 +62,7 @@ bool StreamBase::checkInitialised() { void StreamBase::updateFrameRate(double fps) { frame_mod = 1; - if ( (fps < 0) || !fps ) { + if ( (fps < 0) || !fps || isinf(fps) ) { Debug(1, "Zero or negative fps %f in updateFrameRate. Setting frame_mod=1 and effective_fps=0.0", fps); effective_fps = 0.0; base_fps = 0.0; diff --git a/src/zm_videostore.cpp b/src/zm_videostore.cpp index 30a482f75..849c9575f 100644 --- a/src/zm_videostore.cpp +++ b/src/zm_videostore.cpp @@ -979,13 +979,6 @@ int VideoStore::writeAudioFramePacket(AVPacket *ipkt) { // decoded data Debug(2, "Converting %d to %d samples", in_frame->nb_samples, out_frame->nb_samples); #if defined(HAVE_LIBSWRESAMPLE) -#if 0 - ret = swr_convert(resample_ctx, - out_frame->data, frame_size, - (const uint8_t**)in_frame->data, - in_frame->nb_samples - ); -#else ret = swr_convert_frame(resample_ctx, out_frame, in_frame); av_frame_unref(in_frame); if ( ret < 0 ) { @@ -993,7 +986,6 @@ int VideoStore::writeAudioFramePacket(AVPacket *ipkt) { av_make_error_string(ret).c_str()); return 0; } -#endif if ((ret = av_audio_fifo_realloc(fifo, av_audio_fifo_size(fifo) + out_frame->nb_samples)) < 0) { Error("Could not reallocate FIFO"); return 0; @@ -1031,7 +1023,6 @@ int VideoStore::writeAudioFramePacket(AVPacket *ipkt) { return 0; } - #if defined(HAVE_LIBAVRESAMPLE) int samples_available = avresample_available(resample_ctx); if ( samples_available < frame_size ) { Debug(1, "Not enough samples yet (%d)", samples_available);