From 3b9b49c44d540f5b6071288b0b8a5ad23126fdb5 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Mon, 2 Dec 2019 15:51:03 -0500 Subject: [PATCH 1/2] wip work on fixing zms mp4 to mjpeg --- src/zm_ffmpeg_input.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/zm_ffmpeg_input.cpp b/src/zm_ffmpeg_input.cpp index ab7cbe8d3..ea21d82be 100644 --- a/src/zm_ffmpeg_input.cpp +++ b/src/zm_ffmpeg_input.cpp @@ -102,7 +102,7 @@ int FFmpeg_Input::Open(const char *filepath) { } // end int FFmpeg_Input::Open( const char * filepath ) AVFrame *FFmpeg_Input::get_frame(int stream_id) { - Debug(1, "Getting frame from stream %d", stream_id); + Debug(4, "Getting frame from stream %d", stream_id); int frameComplete = false; AVPacket packet; @@ -138,9 +138,9 @@ AVFrame *FFmpeg_Input::get_frame(int stream_id) { frame = zm_av_frame_alloc(); } ret = zm_send_packet_receive_frame(context, frame, packet); - if ( ret <= 0 ) { - Error("Unable to decode frame at frame %d: %s, continuing", - streams[packet.stream_index].frame_count, av_make_error_string(ret).c_str()); + if ( ret < 0 ) { + Error("Unable to decode frame at frame %d: %d %s, continuing", + streams[packet.stream_index].frame_count, ret, av_make_error_string(ret).c_str()); zm_av_packet_unref(&packet); av_frame_free(&frame); continue; @@ -175,7 +175,9 @@ AVFrame *FFmpeg_Input::get_frame(int stream_id, double at) { } // Have to grab a frame to update our current frame to know where we are get_frame(stream_id); - } // end if ! frame + zm_dump_frame(frame, "Got first frame, returning it"); + return frame; + } // end if !frame if ( frame->pts > seek_target ) { zm_dump_frame(frame, "frame->pts > seek_target, seek backwards"); @@ -192,12 +194,15 @@ AVFrame *FFmpeg_Input::get_frame(int stream_id, double at) { } // end if frame->pts > seek_target // Seeking seems to typically seek to a keyframe, so then we have to decode until we get the frame we want. - if ( frame->pts <= seek_target ) { + if ( frame->pts <= seek_target ) { zm_dump_frame(frame, "pts <= seek_target"); while ( frame && (frame->pts < seek_target) ) { - if ( !get_frame(stream_id) ) + if ( !get_frame(stream_id) ) { + Warning("Got no frame. returning nothing"); return frame; + } } + zm_dump_frame(frame, "frame->pts <= seek_target, got"); return frame; } From 9972a2fd0c686529bfda11523687130c9fe312a9 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Mon, 2 Dec 2019 15:51:08 -0500 Subject: [PATCH 2/2] wip work on fixing zms mp4 to mjpeg --- src/zm_eventstream.cpp | 193 +++++++++++++++++++++++------------------ src/zm_eventstream.h | 2 +- 2 files changed, 111 insertions(+), 84 deletions(-) diff --git a/src/zm_eventstream.cpp b/src/zm_eventstream.cpp index 2ca12972d..f9a4b5d9c 100644 --- a/src/zm_eventstream.cpp +++ b/src/zm_eventstream.cpp @@ -244,7 +244,7 @@ bool EventStream::loadEventData(uint64_t event_id) { event_data->frames[i-1].timestamp = last_timestamp + ((i-last_id)*frame_delta); event_data->frames[i-1].offset = event_data->frames[i-1].timestamp - event_data->start_time; event_data->frames[i-1].in_db = false; - Debug(3,"Frame %d timestamp:(%f), offset(%f) delta(%f), in_db(%d)", + Debug(3, "Frame %d timestamp:(%f), offset(%f) delta(%f), in_db(%d)", i, event_data->frames[i-1].timestamp, event_data->frames[i-1].offset, @@ -260,7 +260,7 @@ bool EventStream::loadEventData(uint64_t event_id) { last_id = id; last_delta = delta; last_timestamp = event_data->frames[id-1].timestamp; - Debug(4, "Frame %d timestamp:(%f), offset(%f) delta(%f), in_db(%d)", + Debug(3, "Frame %d timestamp:(%f), offset(%f) delta(%f), in_db(%d)", id, event_data->frames[id-1].timestamp, event_data->frames[id-1].offset, @@ -303,7 +303,7 @@ bool EventStream::loadEventData(uint64_t event_id) { void EventStream::processCommand(const CmdMsg *msg) { Debug(2, "Got message, type %d, msg %d", msg->msg_type, msg->msg_data[0]); // Check for incoming command - switch( (MsgCommand)msg->msg_data[0] ) { + switch ( (MsgCommand)msg->msg_data[0] ) { case CMD_PAUSE : Debug(1, "Got PAUSE command"); @@ -498,6 +498,7 @@ void EventStream::processCommand(const CmdMsg *msg) { // Do nothing, for now break; } + struct { uint64_t event_id; int progress; @@ -535,9 +536,9 @@ void EventStream::processCommand(const CmdMsg *msg) { exit(0); updateFrameRate((double)event_data->frame_count/event_data->duration); -} // void EventStream::processCommand(const CmdMsg *msg) +} // void EventStream::processCommand(const CmdMsg *msg) -void EventStream::checkEventLoaded() { +bool EventStream::checkEventLoaded() { static char sql[ZM_SQL_SML_BUFSIZ]; if ( curr_frame_id <= 0 ) { @@ -552,7 +553,7 @@ void EventStream::checkEventLoaded() { // No event change required Debug(3, "No event change required, as curr frame %d <=> event frames %d", curr_frame_id, event_data->frame_count); - return; + return false; } // Event change required. @@ -586,6 +587,7 @@ void EventStream::checkEventLoaded() { else curr_frame_id = 1; Debug(2, "New frame id = %d", curr_frame_id); + return true; } else { Debug(2, "No next event loaded using %s. Pausing", sql); if ( curr_frame_id <= 0 ) @@ -594,7 +596,7 @@ void EventStream::checkEventLoaded() { curr_frame_id = event_data->frame_count; paused = true; sendTextFrame("No more event data found"); - } // end if found a new event or not + } // end if found a new event or not mysql_free_result(result); forceEventChange = false; } else { @@ -605,7 +607,8 @@ void EventStream::checkEventLoaded() { curr_frame_id = event_data->frame_count; paused = true; } -} // void EventStream::checkEventLoaded() + return false; +} // void EventStream::checkEventLoaded() Image * EventStream::getImage( ) { static char filepath[PATH_MAX]; @@ -650,7 +653,8 @@ bool EventStream::sendFrame(int delta_us) { Image *send_image = prepareImage(&image); if ( !vid_stream ) { - vid_stream = new VideoStream("pipe:", format, bitrate, effective_fps, send_image->Colours(), send_image->SubpixelOrder(), send_image->Width(), send_image->Height()); + vid_stream = new VideoStream("pipe:", format, bitrate, effective_fps, + send_image->Colours(), send_image->SubpixelOrder(), send_image->Width(), send_image->Height()); fprintf(stdout, "Content-type: %s\r\n\r\n", vid_stream->MimeType()); vid_stream->OpenStream(); } @@ -805,6 +809,10 @@ void EventStream::runStream() { updateFrameRate((double)event_data->frame_count/event_data->duration); gettimeofday(&start, NULL); uint64_t start_usec = start.tv_sec * 1000000 + start.tv_usec; + uint64_t last_frame_offset = 0; + + bool in_event = true; + double time_to_event = 0; while ( !zm_terminate ) { gettimeofday(&now, NULL); @@ -829,77 +837,17 @@ void EventStream::runStream() { Debug(2, "Not checking command queue"); } - // Get current frame data FrameData *frame_data = &event_data->frames[curr_frame_id-1]; - //Info( "cst:%.2f", curr_stream_time ); - //Info( "cfid:%d", curr_frame_id ); - //Info( "fdt:%d", frame_data->timestamp ); if ( !paused ) { - Debug(3, "Not paused at frame %d", curr_frame_id); - - // This next bit is to determine if we are in the current event time wise - // and whether to show an image saying how long until the next event. - bool in_event = true; - double time_to_event = 0; - if ( replay_rate > 0 ) { - time_to_event = event_data->frames[0].timestamp - curr_stream_time; - if ( time_to_event > 0 ) - in_event = false; - } else if ( replay_rate < 0 ) { - time_to_event = curr_stream_time - event_data->frames[event_data->frame_count-1].timestamp; - if ( time_to_event > 0 ) - in_event = false; - } - Debug(1, "replay rate(%d) in_event(%d) time_to_event(%f)=curr_stream_time(%f)-frame timestamp:%f", - replay_rate, in_event, time_to_event, curr_stream_time, event_data->frames[event_data->frame_count-1].timestamp); - if ( !in_event ) { - double actual_delta_time = TV_2_FLOAT(now) - last_frame_sent; - Debug(1, "Ctual delta time = %f = %f - %f", actual_delta_time , TV_2_FLOAT(now) , last_frame_sent); - // > 1 second - if ( actual_delta_time > 1 ) { - Debug(1, "Sending time to next event frame"); - static char frame_text[64]; - snprintf(frame_text, sizeof(frame_text), "Time to next event = %d seconds", (int)time_to_event); - if ( !sendTextFrame(frame_text) ) - zm_terminate = true; - } else { - Debug(1, "Not Sending time to next event frame because actual delta time is %f", actual_delta_time); - } - //else - //{ - // FIXME ICON But we are not paused. We are somehow still in the event? - double sleep_time = (replay_rate>0?1:-1) * ((1.0L * replay_rate * STREAM_PAUSE_WAIT)/(ZM_RATE_BASE * 1000000)); - //double sleep_time = (replay_rate * STREAM_PAUSE_WAIT)/(ZM_RATE_BASE * 1000000); - //// ZM_RATE_BASE == 100, and 1x replay_rate is 100 - //double sleep_time = ((replay_rate/ZM_RATE_BASE) * STREAM_PAUSE_WAIT)/1000000; - if ( ! sleep_time ) { - sleep_time += STREAM_PAUSE_WAIT/1000000; - } - curr_stream_time += sleep_time; - Debug(2, "Sleeping (%dus) because we are not at the next event yet, adding %f", STREAM_PAUSE_WAIT, sleep_time); - usleep(STREAM_PAUSE_WAIT); - - //curr_stream_time += (1.0L * replay_rate * STREAM_PAUSE_WAIT)/(ZM_RATE_BASE * 1000000); - //} - continue; - } // end if !in_event - // Figure out if we should send this frame - Debug(3, "cur_frame_id (%d-1) mod frame_mod(%d)", curr_frame_id, frame_mod); + Debug(3, "not paused at cur_frame_id (%d-1) mod frame_mod(%d)", curr_frame_id, frame_mod); // If we are streaming and this frame is due to be sent // frame mod defaults to 1 and if we are going faster than max_fps will get multiplied by 2 // so if it is 2, then we send every other frame, if is it 4 then every fourth frame, etc. + if ( (frame_mod == 1) || (((curr_frame_id-1)%frame_mod) == 0) ) { - delta_us = (unsigned int)(frame_data->delta * 1000000); - Debug(3, "frame delta %uus ", delta_us); - // if effective > base we should speed up frame delivery - delta_us = (unsigned int)((delta_us * base_fps)/effective_fps); - Debug(3, "delta %u = base_fps(%f)/effective fps(%f)", delta_us, base_fps, effective_fps); - // but must not exceed maxfps - delta_us = max(delta_us, 1000000 / maxfps); - Debug(3, "delta %u = base_fps(%f)/effective fps(%f) from 30fps", delta_us, base_fps, effective_fps); send_frame = true; } } else if ( step != 0 ) { @@ -908,16 +856,48 @@ void EventStream::runStream() { step = 0; send_frame = true; } else if ( !send_frame ) { - // We are paused, not stepping and doing nothing + // We are paused, not stepping and doing nothing, meaning that comms didn't set send_frame to true double actual_delta_time = TV_2_FLOAT(now) - last_frame_sent; if ( actual_delta_time > MAX_STREAM_DELAY ) { // Send keepalive Debug(2, "Sending keepalive frame"); send_frame = true; - //} else { - //Debug(2, "Not Sending keepalive frame"); } - } // end if streaming stepping or doing nothing + } // end if streaming stepping or doing nothing + + // time_to_event > 0 means that we are not in the event + if ( time_to_event > 0 ) { + double actual_delta_time = TV_2_FLOAT(now) - last_frame_sent; + Debug(1, "Actual delta time = %f = %f - %f", actual_delta_time, TV_2_FLOAT(now), last_frame_sent); + // > 1 second + if ( actual_delta_time > 1 ) { + Debug(1, "Sending time to next event frame"); + static char frame_text[64]; + snprintf(frame_text, sizeof(frame_text), "Time to next event = %d seconds", (int)time_to_event); + if ( !sendTextFrame(frame_text) ) + zm_terminate = true; + } else { + Debug(1, "Not Sending time to next event frame because actual delta time is %f", actual_delta_time); + } + //else + //{ + // FIXME ICON But we are not paused. We are somehow still in the event? + double sleep_time = (replay_rate>0?1:-1) * ((1.0L * replay_rate * STREAM_PAUSE_WAIT)/(ZM_RATE_BASE * 1000000)); + //double sleep_time = (replay_rate * STREAM_PAUSE_WAIT)/(ZM_RATE_BASE * 1000000); + //// ZM_RATE_BASE == 100, and 1x replay_rate is 100 + //double sleep_time = ((replay_rate/ZM_RATE_BASE) * STREAM_PAUSE_WAIT)/1000000; + if ( !sleep_time ) { + sleep_time += STREAM_PAUSE_WAIT/1000000; + } + curr_stream_time += sleep_time; + time_to_event -= sleep_time; + Debug(2, "Sleeping (%dus) because we are not at the next event yet, adding %f", STREAM_PAUSE_WAIT, sleep_time); + usleep(STREAM_PAUSE_WAIT); + + //curr_stream_time += (1.0L * replay_rate * STREAM_PAUSE_WAIT)/(ZM_RATE_BASE * 1000000); + //} + continue; + } // end if !in_event if ( send_frame ) { if ( !sendFrame(delta_us) ) { @@ -929,6 +909,17 @@ void EventStream::runStream() { curr_stream_time = frame_data->timestamp; if ( !paused ) { + + // delta is since the last frame + delta_us = (unsigned int)(frame_data->delta * 1000000); + Debug(3, "frame delta %uus ", delta_us); + // if effective > base we should speed up frame delivery + delta_us = (unsigned int)((delta_us * base_fps)/effective_fps); + Debug(3, "delta %u = base_fps(%f)/effective fps(%f)", delta_us, base_fps, effective_fps); + // but must not exceed maxfps + delta_us = max(delta_us, 1000000/maxfps); + Debug(3, "delta %u = base_fps(%f)/effective fps(%f) from 30fps", delta_us, base_fps, effective_fps); + // +/- 1? What if we are skipping frames? curr_frame_id += (replay_rate>0) ? frame_mod : -1*frame_mod; // sending the frame may have taken some time, so reload now @@ -936,7 +927,12 @@ void EventStream::runStream() { uint64_t now_usec = (now.tv_sec * 1000000 + now.tv_usec); // we incremented by replay_rate, so might have jumped past frame_count - if ( (mode == MODE_SINGLE) && ((unsigned int)curr_frame_id >= event_data->frame_count) ) { + if ( (mode == MODE_SINGLE) && ( + (curr_frame_id < 1 ) + || + ((unsigned int)curr_frame_id >= event_data->frame_count) + ) + ) { Debug(2, "Have mode==MODE_SINGLE and at end of event, looping back to start"); curr_frame_id = 1; // Have to reset start_usec to now when replaying @@ -951,9 +947,21 @@ void EventStream::runStream() { // There are two ways to go about this, not sure which is correct. // you can calculate the relationship between now and the start // or calc the relationship from the last frame. I think from the start is better as it self-corrects + // + if ( last_frame_offset ) { + // We assume that we are going forward and the next frame is in the future. + delta_us = frame_data->offset * 1000000 - (now_usec-start_usec); + // - (now_usec - start_usec); + Debug(2, "New delta_us now %" PRIu64 " - start %" PRIu64 " = %d offset %" PRId64 " - elapsed = %dusec", + now_usec, start_usec, now_usec-start_usec, frame_data->offset * 1000000, delta_us); + } else { + Debug(2, "No last frame_offset, no sleep"); + delta_us = 0; + } + last_frame_offset = frame_data->offset * 1000000; - if ( send_frame && type != STREAM_MPEG ) { - if ( delta_us > 0) { + if ( send_frame && (type != STREAM_MPEG) ) { + if ( delta_us > 0 ) { if ( delta_us > MAX_SLEEP_USEC ) { Debug(1, "Limiting sleep to %d because calculated sleep is too long %d", MAX_SLEEP_USEC, delta_us); delta_us = MAX_SLEEP_USEC; @@ -978,14 +986,33 @@ void EventStream::runStream() { } usleep(delta_us); } + // We are paused, so might be stepping + //if ( step != 0 )// Adding 0 is cheaper than an if 0 + // curr_frame_id starts at 1 though, so we might skip the first frame? + curr_frame_id += step; } // end if !paused - //if ( step != 0 )// Adding 0 is cheaper than an if 0 - // curr_frame_id starts at 1 though, so we might skip the first frame? - curr_frame_id += step; - // Detects when we hit end of event and will load the next event or previous event - checkEventLoaded(); + if ( checkEventLoaded() ) { + // Have change of event + + // This next bit is to determine if we are in the current event time wise + // and whether to show an image saying how long until the next event. + if ( replay_rate > 0 ) { + // This doesn't make sense unless we have hit the end of the event. + time_to_event = event_data->frames[0].timestamp - curr_stream_time; + Debug(1, "replay rate(%d) time_to_event(%f)=frame timestamp:%f - curr_stream_time(%f)", + replay_rate, time_to_event, + event_data->frames[0].timestamp, + curr_stream_time); + + } else if ( replay_rate < 0 ) { + time_to_event = curr_stream_time - event_data->frames[event_data->frame_count-1].timestamp; + Debug(1, "replay rate(%d) time_to_event(%f)=curr_stream_time(%f)-frame timestamp:%f", + replay_rate, time_to_event, curr_stream_time, event_data->frames[event_data->frame_count-1].timestamp); + } + + } } // end while ! zm_terminate #if HAVE_LIBAVCODEC if ( type == STREAM_MPEG ) diff --git a/src/zm_eventstream.h b/src/zm_eventstream.h index 5e7d91bb2..980f2f6e1 100644 --- a/src/zm_eventstream.h +++ b/src/zm_eventstream.h @@ -89,7 +89,7 @@ class EventStream : public StreamBase { bool loadInitialEventData( uint64_t init_event_id, unsigned int init_frame_id ); bool loadInitialEventData( int monitor_id, time_t event_time ); - void checkEventLoaded(); + bool checkEventLoaded(); void processCommand( const CmdMsg *msg ); bool sendFrame( int delta_us );