s.orphanedVideoCheck added
- "config.cron.deleteOrphans" has been removed. It has been replace by a one-time-run at startup with "config.insertOrphans". As the variable name suggests, instead of deleting, it will insert videos found without a database row. - By default "config.orphanedVideoCheckMax" will only check up to 20 video. You can raise this value to any number you choose but be careful as it will check that number of videos on every start. - this function also runs if a camera exits unexpectedly.merge-requests/34/head
Moe
parent
3b05d011df
commit
f2797cdb0b
81
cron.js
81
cron.js
|
|
@ -156,15 +156,15 @@ s.utcToLocal = function(time){
|
|||
s.localToUtc = function(time){
|
||||
return moment(time).utc()
|
||||
}
|
||||
s.nameToTime=function(x){x=x.replace('.webm','').replace('.mp4','').split('T'),x[1]=x[1].replace(/-/g,':');x=x.join(' ');return x;}
|
||||
s.nameToTime = function(x){x=x.replace('.webm','').replace('.mp4','').split('T'),x[1]=x[1].replace(/-/g,':');x=x.join(' ');return x;}
|
||||
io = require('socket.io-client')('ws://'+config.ip+':'+config.port);//connect to master
|
||||
s.cx=function(x){x.cronKey=config.cron.key;return io.emit('cron',x)}
|
||||
s.cx = function(x){x.cronKey=config.cron.key;return io.emit('cron',x)}
|
||||
//emulate master socket emitter
|
||||
s.tx=function(x,y){s.cx({f:'s.tx',data:x,to:y})}
|
||||
s.deleteVideo=function(x){s.cx({f:'s.deleteVideo',file:x})}
|
||||
s.tx = function(x,y){s.cx({f:'s.tx',data:x,to:y})}
|
||||
s.deleteVideo = function(x){s.cx({f:'s.deleteVideo',file:x})}
|
||||
//Cron Job
|
||||
s.cx({f:'init',time:moment()})
|
||||
s.getVideoDirectory=function(e){
|
||||
s.getVideoDirectory = function(e){
|
||||
if(e.mid&&!e.id){e.id=e.mid};
|
||||
if(e.details&&(e.details instanceof Object)===false){
|
||||
try{e.details=JSON.parse(e.details)}catch(err){}
|
||||
|
|
@ -175,13 +175,13 @@ s.getVideoDirectory=function(e){
|
|||
return s.dir.videos+e.ke+'/'+e.id+'/';
|
||||
}
|
||||
}
|
||||
s.getFileBinDirectory=function(e){
|
||||
s.getFileBinDirectory = function(e){
|
||||
if(e.mid&&!e.id){e.id=e.mid};
|
||||
return s.dir.fileBin+e.ke+'/'+e.id+'/';
|
||||
}
|
||||
//filters set by the user in their dashboard
|
||||
//deleting old videos is part of the filter - config.cron.deleteOld
|
||||
s.checkFilterRules=function(v,callback){
|
||||
s.checkFilterRules = function(v,callback){
|
||||
//filters
|
||||
if(!v.d.filters||v.d.filters==''){
|
||||
v.d.filters={};
|
||||
|
|
@ -289,7 +289,7 @@ s.checkFilterRules=function(v,callback){
|
|||
}
|
||||
}
|
||||
//database rows with no videos in the filesystem
|
||||
s.deleteRowsWithNoVideo=function(v,callback){
|
||||
s.deleteRowsWithNoVideo = function(v,callback){
|
||||
if(
|
||||
config.cron.deleteNoVideo===true&&(
|
||||
config.cron.deleteNoVideoRecursion===true||
|
||||
|
|
@ -338,7 +338,7 @@ s.deleteRowsWithNoVideo=function(v,callback){
|
|||
}
|
||||
}
|
||||
//info about what the application is doing
|
||||
s.deleteOldLogs=function(v,callback){
|
||||
s.deleteOldLogs = function(v,callback){
|
||||
if(!v.d.log_days||v.d.log_days==''){v.d.log_days=10}else{v.d.log_days=parseFloat(v.d.log_days)};
|
||||
if(config.cron.deleteLogs===true&&v.d.log_days!==0){
|
||||
s.sqlQuery("DELETE FROM Logs WHERE ke=? AND `time` < "+s.sqlDate('? DAYS'),[v.ke,v.d.log_days],function(err,rrr){
|
||||
|
|
@ -353,7 +353,7 @@ s.deleteOldLogs=function(v,callback){
|
|||
}
|
||||
}
|
||||
//events - motion, object, etc. detections
|
||||
s.deleteOldEvents=function(v,callback){
|
||||
s.deleteOldEvents = function(v,callback){
|
||||
if(!v.d.event_days||v.d.event_days==''){v.d.event_days=10}else{v.d.event_days=parseFloat(v.d.event_days)};
|
||||
if(config.cron.deleteEvents===true&&v.d.event_days!==0){
|
||||
s.sqlQuery("DELETE FROM Events WHERE ke=? AND `time` < "+s.sqlDate('? DAYS'),[v.ke,v.d.event_days],function(err,rrr){
|
||||
|
|
@ -368,7 +368,7 @@ s.deleteOldEvents=function(v,callback){
|
|||
}
|
||||
}
|
||||
//check for temporary files (special archive)
|
||||
s.deleteOldFileBins=function(v,callback){
|
||||
s.deleteOldFileBins = function(v,callback){
|
||||
if(!v.d.fileBin_days||v.d.fileBin_days==''){v.d.fileBin_days=10}else{v.d.fileBin_days=parseFloat(v.d.fileBin_days)};
|
||||
if(config.cron.deleteFileBins===true&&v.d.fileBin_days!==0){
|
||||
var fileBinQuery = " FROM Files WHERE ke=? AND `time` < "+s.sqlDate('? DAYS');
|
||||
|
|
@ -397,60 +397,12 @@ s.deleteOldFileBins=function(v,callback){
|
|||
}
|
||||
}
|
||||
//check for files with no database row
|
||||
s.checkForOrphanedFiles=function(v,callback){
|
||||
if(config.cron.deleteOrphans===true){
|
||||
var finish=function(count){
|
||||
if(count>0 || config.debugLog === true){
|
||||
s.cx({f:'deleteOrphanedFiles',msg:count+' SQL rows with no database row deleted',ke:v.ke,time:moment()})
|
||||
}
|
||||
callback()
|
||||
}
|
||||
e={};
|
||||
var numberOfItems = 0;
|
||||
s.sqlQuery('SELECT * FROM Monitors WHERE ke=?',[v.ke],function(arr,b) {
|
||||
if(b&&b[0]){
|
||||
b.forEach(function(mon,m){
|
||||
fs.readdir(s.getVideoDirectory(mon), function(err, items) {
|
||||
e.query=[];
|
||||
e.filesFound=[mon.ke,mon.mid];
|
||||
numberOfItems+=items.length;
|
||||
if(items&&items.length>0){
|
||||
items.forEach(function(v,n){
|
||||
e.query.push('time=?')
|
||||
e.filesFound.push(s.nameToTime(v))
|
||||
})
|
||||
s.sqlQuery('SELECT * FROM Videos WHERE ke=? AND mid=? AND ('+e.query.join(' OR ')+')',e.filesFound,function(arr,r) {
|
||||
if(!r){r=[]};
|
||||
e.foundSQLrows=[];
|
||||
r.forEach(function(v,n){
|
||||
v.index=e.filesFound.indexOf(s.moment(v.time,'YYYY-MM-DD HH:mm:ss'));
|
||||
if(v.index>-1){
|
||||
delete(items[v.index-2]);
|
||||
}
|
||||
});
|
||||
items.forEach(function(v,n){
|
||||
if(v&&v!==null){
|
||||
exec('rm '+s.getVideoDirectory(mon)+v);
|
||||
}
|
||||
if(m===b.length-1&&n===items.length-1){
|
||||
finish(numberOfItems)
|
||||
}
|
||||
})
|
||||
})
|
||||
}else{
|
||||
if(m===b.length-1){
|
||||
finish(numberOfItems)
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
}else{
|
||||
finish(numberOfItems)
|
||||
}
|
||||
});
|
||||
}else{
|
||||
callback()
|
||||
s.checkForOrphanedFiles = function(v,callback){
|
||||
if(config.cron.deleteOrphans === true){
|
||||
console.log('"config.cron.deleteOrphans" has been removed. It has been replace by a one-time-run at startup with "config.insertOrphans". As the variable name suggests, instead of deleting, it will insert videos found without a database row.')
|
||||
console.log('By default "config.orphanedVideoCheckMax" will only check up to 20 video. You can raise this value to any number you choose but be careful as it will check that number of videos on every start.')
|
||||
}
|
||||
callback()
|
||||
}
|
||||
//user processing function
|
||||
s.processUser = function(number,rows){
|
||||
|
|
@ -516,7 +468,6 @@ s.processUser = function(number,rows){
|
|||
s.deleteRowsWithNoVideo(v,function(){
|
||||
s.debugLog('--- deleteRowsWithNoVideo Complete')
|
||||
s.checkForOrphanedFiles(v,function(){
|
||||
s.debugLog('--- checkForOrphanedFiles Complete')
|
||||
//done user, unlock current, and do next
|
||||
s.overlapLock[v.ke]=false;
|
||||
s.processUser(number+1,rows)
|
||||
|
|
|
|||
|
|
@ -637,6 +637,7 @@
|
|||
"startUpText3": "waiting to give unfinished video check some time. 3 seconds.",
|
||||
"startUpText4": "Starting Monitors",
|
||||
"startUpText5": "Shinobi is ready.",
|
||||
"startUpText6": "Orphaned Videos Found and Inserted",
|
||||
"Migrator": "Migrator",
|
||||
"Host Type": "Host Type",
|
||||
"Edit": "Edit",
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ module.exports = function(s){
|
|||
if(config.cron === undefined)config.cron={};
|
||||
if(config.cron.enabled === undefined)config.cron.enabled=true;
|
||||
if(config.cron.deleteOld === undefined)config.cron.deleteOld=true;
|
||||
if(config.cron.deleteOrphans === undefined)config.cron.deleteOrphans=false;
|
||||
if(config.cron.deleteNoVideo === undefined)config.cron.deleteNoVideo=true;
|
||||
if(config.cron.deleteNoVideoRecursion === undefined)config.cron.deleteNoVideoRecursion=false;
|
||||
if(config.cron.deleteOverMax === undefined)config.cron.deleteOverMax=true;
|
||||
|
|
@ -37,6 +36,8 @@ module.exports = function(s){
|
|||
if(config.iconURL === undefined){config.iconURL = "https://shinobi.video/libs/assets/icon/apple-touch-icon-152x152.png"}
|
||||
if(config.pipeAddition === undefined){config.pipeAddition=7}else{config.pipeAddition=parseInt(config.pipeAddition)}
|
||||
if(config.hideCloudSaveUrls === undefined){config.hideCloudSaveUrls = true}
|
||||
if(config.insertOrphans === undefined){config.insertOrphans = true}
|
||||
if(config.orphanedVideoCheckMax === undefined){config.orphanedVideoCheckMax = 20}
|
||||
//Child Nodes
|
||||
if(config.childNodes === undefined)config.childNodes = {};
|
||||
//enabled
|
||||
|
|
|
|||
|
|
@ -606,6 +606,7 @@ module.exports = function(s,config,lang){
|
|||
s.launchMonitorProcesses(e);
|
||||
s.sendMonitorStatus({id:e.id,ke:e.ke,status:lang.Restarting});
|
||||
s.userLog(e,{type:lang['Camera is not recording'],msg:{msg:lang['Restarting Process']}});
|
||||
s.orphanedVideoCheck(e,2,null,true)
|
||||
}
|
||||
},60000 * cutoff * 1.1);
|
||||
}
|
||||
|
|
@ -615,6 +616,7 @@ module.exports = function(s,config,lang){
|
|||
if(s.group[e.ke].mon[e.id].isStarted === true){
|
||||
s.launchMonitorProcesses(e);
|
||||
s.userLog(e,{type:lang['Camera is not streaming'],msg:{msg:lang['Restarting Process']}});
|
||||
s.orphanedVideoCheck(e,2,null,true)
|
||||
}
|
||||
},60000*1);
|
||||
}
|
||||
|
|
@ -725,6 +727,7 @@ module.exports = function(s,config,lang){
|
|||
s.userLog(e,{type:lang['Process Unexpected Exit'],msg:{msg:lang['Process Crashed for Monitor'],cmd:s.group[e.ke].mon[e.id].ffmpeg}});
|
||||
}
|
||||
s.fatalCameraError(e,'Process Unexpected Exit');
|
||||
s.orphanedVideoCheck(e,2,null,true)
|
||||
}
|
||||
}
|
||||
s.group[e.ke].mon[e.id].spawn.on('end',s.group[e.ke].mon[e.id].spawn_exit)
|
||||
|
|
@ -969,7 +972,8 @@ module.exports = function(s,config,lang){
|
|||
fs.stat(e.sdir+'s.jpg',function(err,snap){
|
||||
var notStreaming = function(){
|
||||
s.launchMonitorProcesses(e)
|
||||
s.userLog(e,{type:lang['Camera is not streaming'],msg:{msg:lang['Restarting Process']}});
|
||||
s.userLog(e,{type:lang['Camera is not streaming'],msg:{msg:lang['Restarting Process']}})
|
||||
s.orphanedVideoCheck(e,2,null,true)
|
||||
}
|
||||
if(err){
|
||||
notStreaming()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
var fs = require('fs');
|
||||
var moment = require('moment');
|
||||
var crypto = require('crypto');
|
||||
var exec = require('child_process').exec;
|
||||
|
|
@ -18,11 +19,14 @@ module.exports = function(s,config,lang,io){
|
|||
if(monitors && monitors[0]){
|
||||
monitors.forEach(function(monitor){
|
||||
s.initiateMonitorObject(monitor)
|
||||
monitor.details = JSON.parse(monitor.details)
|
||||
s.group[monitor.ke].mon_conf[monitor.mid] = monitor
|
||||
var monObj = Object.assign(monitor,{id : monitor.mid})
|
||||
s.camera(monitor.mode,monObj)
|
||||
});
|
||||
s.orphanedVideoCheck(monitor,null,function(orphanedFilesCount){
|
||||
if(orphanedFilesCount)s.systemLog(monitor.ke+' : '+monitor.mid+' : '+lang.startUpText6+' : '+orphanedFilesCount)
|
||||
monitor.details = monitor.details
|
||||
s.group[monitor.ke].mon_conf[monitor.mid] = monitor
|
||||
var monObj = Object.assign(monitor,{id : monitor.mid})
|
||||
s.camera(monitor.mode,monObj)
|
||||
})
|
||||
})
|
||||
}
|
||||
callback()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -50,8 +50,8 @@ module.exports = function(s,config,lang){
|
|||
}
|
||||
//on video completion
|
||||
s.insertCompletedVideo = function(e,k,callback){
|
||||
//e = video object
|
||||
//k = temporary values
|
||||
//e = monitor object
|
||||
//k = video insertion object
|
||||
s.checkDetails(e)
|
||||
if(!k)k={};
|
||||
e.dir = s.getVideoDirectory(e)
|
||||
|
|
@ -252,4 +252,54 @@ module.exports = function(s,config,lang){
|
|||
}
|
||||
})
|
||||
}
|
||||
s.orphanedVideoCheck = function(monitor,checkMax,callback,forceCheck){
|
||||
var finish = function(orphanedFilesCount){
|
||||
if(callback)callback(orphanedFilesCount)
|
||||
}
|
||||
if(forceCheck === true || config.insertOrphans === true){
|
||||
if(!checkMax){
|
||||
checkMax = config.orphanedVideoCheckMax
|
||||
}
|
||||
var videosDirectory = s.getVideoDirectory(monitor)// + s.formattedTime(video.time) + '.' + video.ext
|
||||
fs.readdir(videosDirectory,function(err,files){
|
||||
if(files && files.length > 0){
|
||||
var fiveRecentFiles = files.sort().slice(0,config.orphanedVideoCheckMax)
|
||||
var completedFile = 0
|
||||
var orphanedFilesCount = 0
|
||||
var fileComplete = function(){
|
||||
++completedFile
|
||||
if(fiveRecentFiles.length === completedFile){
|
||||
finish(orphanedFilesCount)
|
||||
}
|
||||
}
|
||||
fiveRecentFiles.forEach(function(filename){
|
||||
if(/T[0-9][0-9]-[0-9][0-9]-[0-9][0-9]./.test(filename)){
|
||||
var queryValues = [
|
||||
monitor.ke,
|
||||
monitor.mid,
|
||||
s.nameToTime(filename)
|
||||
]
|
||||
s.sqlQuery('SELECT * FROM Videos WHERE ke=? AND mid=? AND time=? LIMIT 1',queryValues,function(err,rows){
|
||||
if(!err && (!rows || !rows[0])){
|
||||
++orphanedFilesCount
|
||||
var video = rows[0]
|
||||
s.insertCompletedVideo(monitor,{
|
||||
file : filename
|
||||
},function(){
|
||||
fileComplete()
|
||||
})
|
||||
}else{
|
||||
fileComplete()
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}else{
|
||||
finish()
|
||||
}
|
||||
})
|
||||
}else{
|
||||
finish()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue