ZoneMinder
/
zoneminder
mirror of https://github.com/ZoneMinder/zoneminder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix issues with too much audio in events by storing packets in the queue with their timestamps converted to AV_TIME_BASE_Q, so that we can sort video and audio packets together.

pull/2618/head
Isaac Connor 2019-06-11 10:19:42 -04:00
parent 3a91880e2e
commit acb95709e6
6 changed files with 91 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/zm_ffmpeg_camera.cpp
View File

@ -873,6 +873,7 @@ int FfmpegCamera::CaptureAndRecord( Image &image, timeval recording, char* event
}
} // end if recording or not
// Buffer video packets, since we are not recording.
// All audio packets are keyframes, so only if it's a video keyframe
if ( packet.stream_index == mVideoStreamId ) {
@ -885,17 +886,18 @@ int FfmpegCamera::CaptureAndRecord( Image &image, timeval recording, char* event
}
packetqueue->clearQueue(monitor->GetPreEventCount(), mVideoStreamId);
packetqueue->queuePacket(&packet);
packetqueue->queuePacket(&packet, mFormatContext->streams[packet.stream_index]);
} else if ( packetqueue->size() ) {
// it's a keyframe or we already have something in the queue
packetqueue->queuePacket(&packet);
packetqueue->queuePacket(&packet, mFormatContext->streams[packet.stream_index]);
}
} else if ( packet.stream_index == mAudioStreamId ) {
// The following lines should ensure that the queue always begins with a video keyframe
//Debug(2, "Have audio packet, reocrd_audio is (%d) and packetqueue.size is (%d)", record_audio, packetqueue.size() );
if ( record_audio && packetqueue->size() ) {
// if it's audio, and we are doing audio, and there is already something in the queue
packetqueue->queuePacket(&packet);
packetqueue->queuePacket(&packet, mFormatContext->streams[packet.stream_index]);
}
} // end if packet type
@ -904,9 +906,19 @@ int FfmpegCamera::CaptureAndRecord( Image &image, timeval recording, char* event
if ( have_video_keyframe || keyframe ) {
if ( videoStore ) {
AVPacket out_packet;
av_init_packet(&out_packet);
if ( zm_av_packet_ref( &out_packet, &packet ) < 0 ) {
Error("error refing packet");
}
out_packet.pts = av_rescale_q(out_packet.pts, mFormatContext->streams[packet.stream_index]->time_base, AV_TIME_BASE_Q);
out_packet.dts = av_rescale_q(out_packet.dts, mFormatContext->streams[packet.stream_index]->time_base, AV_TIME_BASE_Q);
out_packet.duration = av_rescale_q(out_packet.duration, mFormatContext->streams[packet.stream_index]->time_base, AV_TIME_BASE_Q);
//Write the packet to our video store
int ret = videoStore->writeVideoFramePacket(&packet);
int ret = videoStore->writeVideoFramePacket(&out_packet);
zm_av_packet_unref(&out_packet);
if ( ret < 0 ) { //Less than zero and we skipped a frame
zm_av_packet_unref(&packet);
return 0;
@ -1005,15 +1017,26 @@ int FfmpegCamera::CaptureAndRecord( Image &image, timeval recording, char* event
if ( videoStore ) {
if ( record_audio ) {
if ( have_video_keyframe ) {
Debug(3, "Recording audio packet streamindex(%d) packetstreamindex(%d)", mAudioStreamId, packet.stream_index );
//Write the packet to our video store
//FIXME no relevance of last key frame
int ret = videoStore->writeAudioFramePacket( &packet );
if ( ret < 0 ) {//Less than zero and we skipped a frame
Warning("Failure to write audio packet.");
zm_av_packet_unref( &packet );
return 0;
}
AVPacket out_packet;
av_init_packet(&out_packet);
if ( zm_av_packet_ref( &out_packet, &packet ) < 0 ) {
Error("error refing packet");
}
out_packet.pts = av_rescale_q(out_packet.pts, mFormatContext->streams[packet.stream_index]->time_base, AV_TIME_BASE_Q);
out_packet.dts = av_rescale_q(out_packet.dts, mFormatContext->streams[packet.stream_index]->time_base, AV_TIME_BASE_Q);
out_packet.duration = av_rescale_q(out_packet.duration, mFormatContext->streams[packet.stream_index]->time_base, AV_TIME_BASE_Q);
Debug(3, "Recording audio packet streamindex(%d) packetstreamindex(%d)", mAudioStreamId, packet.stream_index);
//Write the packet to our video store
//FIXME no relevance of last key frame
int ret = videoStore->writeAudioFramePacket(&out_packet);
zm_av_packet_unref(&out_packet);
if ( ret < 0 ) {//Less than zero and we skipped a frame
Warning("Failure to write audio packet.");
zm_av_packet_unref( &packet );
return 0;
}
} else {
Debug(3, "Not recording audio yet because we don't have a video keyframe yet");
}

10
src/zm_packet.cpp
View File

@ -24,23 +24,29 @@
using namespace std;
ZMPacket::ZMPacket( AVPacket *p ) {
ZMPacket::ZMPacket( AVPacket *p, AVStream *stream ) {
frame = NULL;
image = NULL;
av_init_packet( &packet );
if ( zm_av_packet_ref( &packet, p ) < 0 ) {
Error("error refing packet");
}
packet.pts = av_rescale_q(packet.pts, stream->time_base, AV_TIME_BASE_Q);
packet.dts = av_rescale_q(packet.dts, stream->time_base, AV_TIME_BASE_Q);
packet.duration = av_rescale_q(packet.duration, stream->time_base, AV_TIME_BASE_Q);
gettimeofday( &timestamp, NULL );
}
ZMPacket::ZMPacket( AVPacket *p, struct timeval *t ) {
ZMPacket::ZMPacket( AVPacket *p, AVStream *stream, struct timeval *t ) {
frame = NULL;
image = NULL;
av_init_packet( &packet );
if ( zm_av_packet_ref( &packet, p ) < 0 ) {
Error("error refing packet");
}
packet.pts = av_rescale_q(packet.pts, stream->time_base, AV_TIME_BASE_Q);
packet.dts = av_rescale_q(packet.pts, stream->time_base, AV_TIME_BASE_Q);
packet.duration = av_rescale_q(packet.pts, stream->time_base, AV_TIME_BASE_Q);
timestamp = *t;
}

4
src/zm_packet.h
View File

@ -38,8 +38,8 @@ class ZMPacket {
struct timeval timestamp;
public:
AVPacket *av_packet() { return &packet; }
ZMPacket( AVPacket *packet, struct timeval *timestamp );
explicit ZMPacket( AVPacket *packet );
ZMPacket( AVPacket *packet, AVStream *stream, struct timeval *timestamp );
explicit ZMPacket( AVPacket *packet, AVStream * );
~ZMPacket();
};

22
src/zm_packetqueue.cpp
View File

@ -57,8 +57,8 @@ bool zm_packetqueue::queuePacket(ZMPacket* zm_packet) {
Debug(2, "Looking at packet with stream index (%d) with dts %" PRId64,
av_packet->stream_index, av_packet->dts);
if (
( av_packet->stream_index == zm_packet->packet.stream_index )
&&
//( av_packet->stream_index == zm_packet->packet.stream_index )
//&&
( av_packet->dts != AV_NOPTS_VALUE )
&&
( av_packet->dts <= zm_packet->packet.dts)
@ -77,7 +77,7 @@ bool zm_packetqueue::queuePacket(ZMPacket* zm_packet) {
//Debug(2, "Found packet with stream index (%d) with dts %" PRId64,
//(*it)->packet.stream_index, (*it)->packet.dts);
if ( it == pktQueue.rbegin() ) {
Debug(2,"Inserting packet with dts %" PRId64 " at end", zm_packet->packet.dts);
Debug(2, "Inserting packet with dts %" PRId64 " at end", zm_packet->packet.dts);
// No dts value, can't so much with it
pktQueue.push_back(zm_packet);
packet_counts[zm_packet->packet.stream_index] += 1;
@ -85,16 +85,22 @@ bool zm_packetqueue::queuePacket(ZMPacket* zm_packet) {
}
// Convert to a forward iterator so that we can insert at end
std::list<ZMPacket *>::iterator f_it = it.base();
Debug(2, "Insert packet with stream index (%d) with dts %" PRId64 " for dts %" PRId64,
(*f_it)->packet.stream_index, (*f_it)->packet.dts, zm_packet->packet.dts);
if ( f_it == pktQueue.end() ) {
Debug(2, "Pushing to end");
pktQueue.push_back(zm_packet);
} else {
Debug(2, "Insert packet with stream index (%d) with dts %" PRId64 " for dts %" PRId64,
(*f_it)->packet.stream_index, (*f_it)->packet.dts, zm_packet->packet.dts);
pktQueue.insert(f_it, zm_packet);
pktQueue.insert(f_it, zm_packet);
}
packet_counts[zm_packet->packet.stream_index] += 1;
return true;
}
Debug(1,"Unable to insert packet for stream %d with dts %" PRId64 " into queue.",
Debug(1,"Unable to find a spot for stream %d with dts %" PRId64 ". Sticking on front",
zm_packet->packet.stream_index, zm_packet->packet.dts);
// Must be before any packets in the queue. Stick it at the beginning
pktQueue.push_front(zm_packet);
@ -102,8 +108,8 @@ bool zm_packetqueue::queuePacket(ZMPacket* zm_packet) {
return true;
} // end bool zm_packetqueue::queuePacket(ZMPacket* zm_packet)
bool zm_packetqueue::queuePacket(AVPacket* av_packet) {
ZMPacket *zm_packet = new ZMPacket(av_packet);
bool zm_packetqueue::queuePacket(AVPacket* av_packet, AVStream *stream) {
ZMPacket *zm_packet = new ZMPacket(av_packet, stream);
return queuePacket(zm_packet);
}

4
src/zm_packetqueue.h
View File

@ -33,9 +33,9 @@ class zm_packetqueue {
public:
zm_packetqueue(int max_stream_id);
virtual ~zm_packetqueue();
bool queuePacket(AVPacket* packet, struct timeval *timestamp);
bool queuePacket(AVPacket* packet, AVStream *stream, struct timeval *timestamp);
bool queuePacket(ZMPacket* packet);
bool queuePacket(AVPacket* packet);
bool queuePacket(AVPacket* packet, AVStream *stream);
ZMPacket * popPacket();
bool popVideoPacket(ZMPacket* packet);
bool popAudioPacket(ZMPacket* packet);

46
src/zm_videostore.cpp
View File

@ -860,22 +860,23 @@ int VideoStore::writeVideoFramePacket(AVPacket *ipkt) {
if ( ipkt->duration != AV_NOPTS_VALUE ) {
duration = av_rescale_q(
ipkt->duration,
video_in_stream->time_base,
AV_TIME_BASE_Q,
video_out_stream->time_base);
Debug(1, "duration from ipkt: pts(%" PRId64 ") - last_pts(%" PRId64 ") = (%" PRId64 ") => (%" PRId64 ") (%d/%d) (%d/%d)",
ipkt->pts,
video_last_pts,
ipkt->duration,
duration,
video_in_stream->time_base.num,
video_in_stream->time_base.den,
1,
AV_TIME_BASE,
video_out_stream->time_base.num,
video_out_stream->time_base.den
);
} else {
duration = av_rescale_q(
ipkt->pts - video_last_pts,
video_in_stream->time_base,
AV_TIME_BASE_Q,
//video_in_stream->time_base,
video_out_stream->time_base);
Debug(1, "duration calc: pts(%" PRId64 ") - last_pts(%" PRId64 ") = (%" PRId64 ") => (%" PRId64 ")",
ipkt->pts,
@ -885,7 +886,7 @@ int VideoStore::writeVideoFramePacket(AVPacket *ipkt) {
);
if ( duration <= 0 ) {
// Why are we setting the duration to 1?
duration = ipkt->duration ? ipkt->duration : av_rescale_q(1,video_in_stream->time_base, video_out_stream->time_base);
duration = ipkt->duration ? ipkt->duration : av_rescale_q(1, AV_TIME_BASE_Q, video_out_stream->time_base);
}
}
opkt.duration = duration;
@ -902,7 +903,8 @@ int VideoStore::writeVideoFramePacket(AVPacket *ipkt) {
} else {
opkt.pts = av_rescale_q(
ipkt->pts - video_first_pts,
video_in_stream->time_base,
AV_TIME_BASE_Q,
//video_in_stream->time_base,
video_out_stream->time_base
);
}
@ -929,7 +931,8 @@ int VideoStore::writeVideoFramePacket(AVPacket *ipkt) {
} else {
opkt.dts = av_rescale_q(
ipkt->dts - video_first_dts,
video_in_stream->time_base,
AV_TIME_BASE_Q,
//video_in_stream->time_base,
video_out_stream->time_base
);
Debug(3, "opkt.dts = %" PRId64 " from ipkt->dts(%" PRId64 ") - first_pts(%" PRId64 ")",
@ -1027,6 +1030,7 @@ int VideoStore::writeAudioFramePacket(AVPacket *ipkt) {
// sending AV_NOPTS_VALUE doesn't really work but we seem to get it in ffmpeg 2.8
out_frame->pts = audio_next_pts;
}
// We need to keep track of this due to resampling
audio_next_pts = out_frame->pts + out_frame->nb_samples;
av_init_packet(&opkt);
@ -1087,7 +1091,8 @@ int VideoStore::writeAudioFramePacket(AVPacket *ipkt) {
if ( ipkt->duration && (ipkt->duration != AV_NOPTS_VALUE) ) {
opkt.duration = av_rescale_q(
ipkt->duration,
audio_in_stream->time_base,
AV_TIME_BASE_Q,
//audio_in_stream->time_base,
audio_out_stream->time_base);
}
// Scale the PTS of the outgoing packet to be the correct time base
@ -1099,7 +1104,8 @@ int VideoStore::writeAudioFramePacket(AVPacket *ipkt) {
} else {
opkt.pts = av_rescale_q(
ipkt->pts - audio_first_pts,
audio_in_stream->time_base,
AV_TIME_BASE_Q,
//audio_in_stream->time_base,
audio_out_stream->time_base);
Debug(2, "audio opkt.pts = %" PRId64 " from ipkt->pts(%" PRId64 ") - first_pts(%" PRId64 ")",
opkt.pts, ipkt->pts, audio_first_pts);
@ -1126,7 +1132,8 @@ int VideoStore::writeAudioFramePacket(AVPacket *ipkt) {
} else {
opkt.dts = av_rescale_q(
ipkt->dts - audio_first_dts,
audio_in_stream->time_base,
AV_TIME_BASE_Q,
//audio_in_stream->time_base,
audio_out_stream->time_base);
Debug(2, "opkt.dts = %" PRId64 " from ipkt.dts(%" PRId64 ") - first_dts(%" PRId64 ")",
opkt.dts, ipkt->dts, audio_first_dts);
@ -1176,6 +1183,7 @@ int VideoStore::writeAudioFramePacket(AVPacket *ipkt) {
int VideoStore::resample_audio() {
// Resample the in_frame into the audioSampleBuffer until we process the whole
// decoded data. Note: pts does not survive resampling or converting
// if we ask for less samples than we input, convert_frame will buffer the remainder apparently
#if defined(HAVE_LIBSWRESAMPLE) || defined(HAVE_LIBAVRESAMPLE)
#if defined(HAVE_LIBSWRESAMPLE)
Debug(2, "Converting %d to %d samples using swresample",
@ -1207,7 +1215,7 @@ int VideoStore::resample_audio() {
audio_next_pts = out_frame->pts + out_frame->nb_samples;
#endif
if ((ret = av_audio_fifo_realloc(fifo, av_audio_fifo_size(fifo) + out_frame->nb_samples)) < 0) {
if ( (ret = av_audio_fifo_realloc(fifo, av_audio_fifo_size(fifo) + out_frame->nb_samples)) < 0 ) {
Error("Could not reallocate FIFO");
return 0;
}
@ -1224,6 +1232,7 @@ int VideoStore::resample_audio() {
// AAC requires 1024 samples per encode. Our input tends to be 160, so need to buffer them.
if ( frame_size > av_audio_fifo_size(fifo) ) {
Debug(1, "Not enough samples in the fifo");
return 0;
}
@ -1233,15 +1242,18 @@ int VideoStore::resample_audio() {
}
out_frame->nb_samples = frame_size;
// resampling changes the duration because the timebase is 1/samples
out_frame->pkt_duration = out_frame->nb_samples;
// out_frame->sample_rate;
if ( in_frame->pts != AV_NOPTS_VALUE ) {
out_frame->pkt_duration = av_rescale_q(
in_frame->pkt_duration,
audio_in_stream->time_base,
audio_out_stream->time_base);
//out_frame->pkt_duration = av_rescale_q(
//in_frame->pkt_duration,
//audio_in_stream->time_base,
//audio_out_stream->time_base);
out_frame->pts = av_rescale_q(
in_frame->pts,
audio_in_stream->time_base,
audio_out_stream->time_base);
AV_TIME_BASE_Q,
//audio_in_ctx->time_base,
audio_out_ctx->time_base);
}
#else
#if defined(HAVE_LIBAVRESAMPLE)