ShinobiSystems
/
Shinobi
mirror of https://gitlab.com/Shinobi-Systems/Shinobi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update fatal error handling in monitor launch

matrix-notifications^2
Moe 2023-02-03 10:59:22 -08:00
parent c9d0eb56d1
commit 1fd01db1c3
1 changed files with 29 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
libs/monitor/utils.js
View File

@ -148,6 +148,7 @@ module.exports = (s,config,lang) => {
clearInterval(activeMonitor.getMonitorCpuUsage); clearInterval(activeMonitor.getMonitorCpuUsage);
clearInterval(activeMonitor.objectCountIntervals); clearInterval(activeMonitor.objectCountIntervals);
clearTimeout(activeMonitor.timeoutToRestart) clearTimeout(activeMonitor.timeoutToRestart)
clearTimeout(activeMonitor.fatalErrorTimeout);
delete(activeMonitor.onvifConnection) delete(activeMonitor.onvifConnection)
// if(activeMonitor.onChildNodeExit){ // if(activeMonitor.onChildNodeExit){
// activeMonitor.onChildNodeExit() // activeMonitor.onChildNodeExit()
@ -709,7 +710,7 @@ module.exports = (s,config,lang) => {
clearTimeout(activeMonitor.trigger_timer) clearTimeout(activeMonitor.trigger_timer)
delete(activeMonitor.trigger_timer) delete(activeMonitor.trigger_timer)
clearInterval(activeMonitor.detector_notrigger_timeout) clearInterval(activeMonitor.detector_notrigger_timeout)
clearTimeout(activeMonitor.err_fatal_timeout); clearTimeout(activeMonitor.fatalErrorTimeout);
activeMonitor.isStarted = false activeMonitor.isStarted = false
activeMonitor.isRecording = false activeMonitor.isRecording = false
s.tx({f:'monitor_stopping',mid:monitorId,ke:groupKey,time:s.formattedTime()},'GRP_'+groupKey); s.tx({f:'monitor_stopping',mid:monitorId,ke:groupKey,time:s.formattedTime()},'GRP_'+groupKey);
@ -1119,7 +1120,7 @@ module.exports = (s,config,lang) => {
} }
if(e.details.record_timelapse === '1'){ if(e.details.record_timelapse === '1'){
var timelapseRecordingDirectory = s.getTimelapseFrameDirectory(e) var timelapseRecordingDirectory = s.getTimelapseFrameDirectory(e)
activeMonitor.spawn.stdio[7].on('data',function(data){ activeMonitor.spawn.stdio[7].on('data', async function(data){
var fileStream = activeMonitor.recordTimelapseWriter var fileStream = activeMonitor.recordTimelapseWriter
if(!fileStream){ if(!fileStream){
var currentDate = s.formattedTime(null,'YYYY-MM-DD') var currentDate = s.formattedTime(null,'YYYY-MM-DD')
@ -1219,7 +1220,7 @@ module.exports = (s,config,lang) => {
activeMonitor.spawn.stdout.on('data',frameToStreamPrimary) activeMonitor.spawn.stdout.on('data',frameToStreamPrimary)
} }
if(e.details.stream_channels && e.details.stream_channels !== ''){ if(e.details.stream_channels && e.details.stream_channels !== ''){
e.details.stream_channels.forEach((fields,number) => { s.parseJSON(e.details.stream_channels,{}).forEach((fields,number) => {
attachStreamChannelHandlers({ attachStreamChannelHandlers({
ke: groupKey, ke: groupKey,
mid: monitorId, mid: monitorId,
@ -1285,6 +1286,19 @@ module.exports = (s,config,lang) => {
} }
}) })
} }
async function doFatalErrorCatch(e,d){
const groupKey = e.ke
const monitorId = e.mid || e.id
if(activeMonitor.isStarted === true){
const activeMonitor = getActiveMonitor(groupKey,monitorId)
activeMonitor.isStarted = false
await cameraDestroy(e)
activeMonitor.isStarted = true
fatalError(e,d)
}else{
await cameraDestroy(e)
}
}
function cameraFilterFfmpegLog(e){ function cameraFilterFfmpegLog(e){
var checkLog = function(d,x){return d.indexOf(x)>-1} var checkLog = function(d,x){return d.indexOf(x)>-1}
const groupKey = e.ke const groupKey = e.ke
@ -1323,30 +1337,18 @@ module.exports = (s,config,lang) => {
break; break;
case checkLog(d,'Connection refused'): case checkLog(d,'Connection refused'):
case checkLog(d,'Connection timed out'): case checkLog(d,'Connection timed out'):
//restart
activeMonitor.timeoutToRestart = setTimeout(function(){
s.userLog(e,{type:lang['Connection timed out'],msg:lang['Retrying...']});
fatalError(e,'Connection timed out');
},1000)
break;
case checkLog(d,'Immediate exit requested'): case checkLog(d,'Immediate exit requested'):
await cameraDestroy(e)
activeMonitor.timeoutToRestart = setTimeout(() => {
launchMonitorProcesses(e)
},15000)
break;
case checkLog(d,'mjpeg_decode_dc'): case checkLog(d,'mjpeg_decode_dc'):
case checkLog(d,'bad vlc'): case checkLog(d,'bad vlc'):
case checkLog(d,'does not contain an image sequence pattern or a pattern is invalid.'):
case checkLog(d,'error dc'): case checkLog(d,'error dc'):
await cameraDestroy(e) // activeMonitor.timeoutToRestart = setTimeout(() => {
activeMonitor.timeoutToRestart = setTimeout(() => { // doFatalErrorCatch(e,d)
launchMonitorProcesses(e) // },15000)
},15000)
break; break;
case checkLog(d,'No route to host'): case checkLog(d,'No route to host'):
await cameraDestroy(e) activeMonitor.timeoutToRestart = setTimeout(async () => {
activeMonitor.timeoutToRestart = setTimeout(() => { doFatalErrorCatch(e,d)
launchMonitorProcesses(e)
},60000) },60000)
break; break;
} }
@ -1425,6 +1427,7 @@ module.exports = (s,config,lang) => {
const monitorConfig = theGroup.rawMonitorConfigurations[monitorId] const monitorConfig = theGroup.rawMonitorConfigurations[monitorId]
const doPingTest = e.type !== 'socket' && e.type !== 'dashcam' && e.protocol !== 'udp' && e.type !== 'local' && e.details.skip_ping !== '1'; const doPingTest = e.type !== 'socket' && e.type !== 'dashcam' && e.protocol !== 'udp' && e.type !== 'local' && e.details.skip_ping !== '1';
const startMonitorInQueue = theGroup.startMonitorInQueue const startMonitorInQueue = theGroup.startMonitorInQueue
if(!activeMonitor.isStarted)return;
// e = monitor object // e = monitor object
clearTimeout(activeMonitor.resetFatalErrorCountTimer) clearTimeout(activeMonitor.resetFatalErrorCountTimer)
activeMonitor.resetFatalErrorCountTimer = setTimeout(()=>{ activeMonitor.resetFatalErrorCountTimer = setTimeout(()=>{
@ -1601,17 +1604,18 @@ module.exports = (s,config,lang) => {
const groupKey = e.ke const groupKey = e.ke
const monitorId = e.mid || e.id const monitorId = e.mid || e.id
const activeMonitor = getActiveMonitor(groupKey,monitorId) const activeMonitor = getActiveMonitor(groupKey,monitorId)
const monitorDetails = getMonitorConfiguration(groupKey,monitorId).details const monitorConfig = copyMonitorConfiguration(groupKey,monitorId)
const monitorDetails = monitorConfig.details
const maxCount = !monitorDetails.fatal_max || isNaN(monitorDetails.fatal_max) ? 0 : parseFloat(monitorDetails.fatal_max); const maxCount = !monitorDetails.fatal_max || isNaN(monitorDetails.fatal_max) ? 0 : parseFloat(monitorDetails.fatal_max);
clearTimeout(activeMonitor.err_fatal_timeout); clearTimeout(activeMonitor.fatalErrorTimeout);
++activeMonitor.errorFatalCount; ++activeMonitor.errorFatalCount;
if(activeMonitor.isStarted === true){ if(activeMonitor.isStarted === true){
activeMonitor.err_fatal_timeout = setTimeout(function(){ activeMonitor.fatalErrorTimeout = setTimeout(function(){
if(maxCount !== 0 && activeMonitor.errorFatalCount > maxCount){ if(maxCount !== 0 && activeMonitor.errorFatalCount > maxCount){
s.userLog(e,{type:lang["Fatal Error"],msg:lang.onFatalErrorExit}); s.userLog(e,{type:lang["Fatal Error"],msg:lang.onFatalErrorExit});
s.camera('stop',{mid:monitorId,ke:groupKey}) s.camera('stop',{mid:monitorId,ke:groupKey})
}else{ }else{
launchMonitorProcesses(s.cleanMonitorObject(e)) launchMonitorProcesses(monitorConfig)
}; };
},5000); },5000);
}else{ }else{
@ -1623,7 +1627,6 @@ module.exports = (s,config,lang) => {
status: lang.Died, status: lang.Died,
code: 7 code: 7
}); });
const monitorConfig = copyMonitorConfiguration(groupKey,monitorId)
s.onMonitorDiedExtensions.forEach(function(extender){ s.onMonitorDiedExtensions.forEach(function(extender){
extender(monitorConfig,e) extender(monitorConfig,e)
}) })