ShinobiSystems
/
Shinobi
mirror of https://gitlab.com/Shinobi-Systems/Shinobi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

make all deleteOldVideos actions occur in cron.js, promisify cron funcs

merge-requests/289/head
Moe Alam 2021-02-05 17:36:10 -08:00
parent 3a4a405e1c
commit bf1d78235b
3 changed files with 362 additions and 337 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

253
cron.js
View File

@ -41,6 +41,7 @@ const {
const {
sqlDate,
knexQuery,
knexQueryPromise,
initiateDatabaseEngine
} = require('./libs/sql/utils.js')(s,config)
var theCronInterval = null
@ -108,40 +109,12 @@ const getFileBinDirectory = function(e){
}
//filters set by the user in their dashboard
//deleting old videos is part of the filter - config.cron.deleteOld
const checkFilterRules = function(v,callback){
const checkFilterRules = function(v){
return new Promise((resolve,reject) => {
//filters
if(!v.d.filters||v.d.filters==''){
v.d.filters={};
}
//delete old videos with filter
if(config.cron.deleteOld === true){
var where = [{
"p1":"end",
"p2":"<=",
"p3": sqlDate(v.d.days+" DAY")
}]
//exclude monitors with their own max days
v.monitorsWithMaxKeepDays.forEach(function(mid){
where.push({
"p1":"mid",
"p2":"!=",
"p3":mid,
})
})
v.d.filters.deleteOldVideosByCron={
"id":"deleteOldVideosByCron",
"name":"deleteOldVideosByCron",
"sort_by":"time",
"sort_by_direction":"ASC",
"limit":"",
"enabled":"1",
"archive":"0",
"email":"0",
"delete":"1",
"execute":"",
"where":where
};
}
s.debugLog('Filters')
var keys = Object.keys(v.d.filters)
if(keys.length>0){
@ -201,16 +174,107 @@ const checkFilterRules = function(v,callback){
}
if(current===keys.length-1){
//last filter
callback()
resolve()
}
})
}else{
//no filters
callback()
resolve()
}
})
}
const deleteVideosByDays = async (v,days,addedQueries) => {
const whereQuery = [
['ke','=',v.ke],
['time','<', sqlDate(days+' DAY')],
addedQueries
]
const selectResponse = await knexQueryPromise({
action: "select",
columns: "*",
table: "Videos",
where: whereQuery
})
const videoRows = selectResponse.rows
let affectedRows = 0
if(videoRows.length > 0){
var i;
for (i = 0; i < videoRows.length; i++) {
const row = videoRows[i];
const dir = getVideoDirectory(row)
const filename = formattedTime(row.time) + '.' + row.ext
await fs.promises.unlink(dir + filename)
sendToWebSocket({
f: 'video_delete',
filename: filename + '.' + row.ext,
mid: row.mid,
ke: row.ke,
time: row.time,
end: formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')
},'GRP_' + row.ke)
}
const deleteResponse = await knexQueryPromise({
action: "delete",
table: "Videos",
where: whereQuery
})
affectedRows = deleteResponse.rows.affectedRows
}
return {
ok: true,
affectedRows: affectedRows,
}
}
const deleteOldVideos = async (v) => {
// v = group, admin user
if(config.cron.deleteOld === true){
const daysOldForDeletion = !isNaN(v.d.days) ? parseFloat(v.d.days) : 5
const monitorsIgnored = []
const monitorsResponse = await knexQueryPromise({
action: "select",
columns: "*",
table: "Monitors",
where: [
['ke','=',v.ke],
]
})
const monitorRows = monitorsResponse.rows
var i;
for (i = 0; i < monitorRows.length; i++) {
const monitor = monitorRows[i]
const monitorId = monitor.id
const details = JSON.parse(monitor.details);
const monitorsMaxDaysToKeep = !isNaN(details.max_keep_days) ? parseFloat(details.max_keep_days) : null
if(monitorsMaxDaysToKeep){
const { affectedRows } = await deleteVideosByDays(v,monitorsMaxDaysToKeep,['mid','=',monitorId])
const hasDeletedRows = affectedRows && affectedRows.length > 0;
if(hasDeletedRows || config.debugLog === true){
postMessage({
f: 'deleteOldVideosByMonitorId',
msg: `${affectedRows} SQL rows older than ${monitorsMaxDaysToKeep} days deleted`,
ke: v.ke,
mid: monitorId,
time: moment(),
})
}
monitorsIgnored.push(['mid','!=',monitorId])
}
}
const { affectedRows } = await deleteVideosByDays(v,daysOldForDeletion,monitorsIgnored)
const hasDeletedRows = affectedRows && affectedRows.length > 0;
if(hasDeletedRows || config.debugLog === true){
postMessage({
f: 'deleteOldVideos',
msg: `${affectedRows} SQL rows older than ${daysOldForDeletion} days deleted`,
ke: v.ke,
time: moment(),
})
}
}
}
//database rows with no videos in the filesystem
const deleteRowsWithNoVideo = function(v,callback){
const deleteRowsWithNoVideo = function(v){
return new Promise((resolve,reject) => {
if(
config.cron.deleteNoVideo===true&&(
config.cron.deleteNoVideoRecursion===true||
@ -244,11 +308,7 @@ const deleteRowsWithNoVideo = function(v,callback){
}
}
var dir = getVideoDirectory(ev)
if(details.isUTC === true){
filename = localToUtc(ev.time).format('YYYY-MM-DDTHH-mm-ss')+'.'+ev.ext
}else{
filename = formattedTime(ev.time)+'.'+ev.ext
}
fileExists = fs.existsSync(dir+filename)
if(fileExists !== true){
deleteVideo(ev)
@ -260,15 +320,17 @@ const deleteRowsWithNoVideo = function(v,callback){
}
}
setTimeout(function(){
callback()
resolve()
},3000)
})
}else{
callback()
resolve()
}
})
}
//info about what the application is doing
const deleteOldLogs = function(v,callback){
const deleteOldLogs = function(v){
return new Promise((resolve,reject) => {
if(!v.d.log_days||v.d.log_days==''){v.d.log_days=10}else{v.d.log_days=parseFloat(v.d.log_days)};
if(config.cron.deleteLogs===true&&v.d.log_days!==0){
knexQuery({
@ -279,18 +341,20 @@ const deleteOldLogs = function(v,callback){
['time','<', sqlDate(v.d.log_days+' DAY')],
]
},(err,rrr) => {
callback()
resolve()
if(err)return console.error(err);
if(rrr.affectedRows && rrr.affectedRows.length>0 || config.debugLog === true){
postMessage({f:'deleteLogs',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.log_days+' days deleted',ke:v.ke,time:moment()})
}
})
}else{
callback()
resolve()
}
})
}
//events - motion, object, etc. detections
const deleteOldEvents = function(v,callback){
const deleteOldEvents = function(v){
return new Promise((resolve,reject) => {
if(!v.d.event_days||v.d.event_days==''){v.d.event_days=10}else{v.d.event_days=parseFloat(v.d.event_days)};
if(config.cron.deleteEvents===true&&v.d.event_days!==0){
knexQuery({
@ -301,18 +365,20 @@ const deleteOldEvents = function(v,callback){
['time','<', sqlDate(v.d.event_days+' DAY')],
]
},(err,rrr) => {
callback()
resolve()
if(err)return console.error(err);
if(rrr.affectedRows && rrr.affectedRows.length > 0 || config.debugLog === true){
postMessage({f:'deleteEvents',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.event_days+' days deleted',ke:v.ke,time:moment()})
}
})
}else{
callback()
resolve()
}
})
}
//event counts
const deleteOldEventCounts = function(v,callback){
const deleteOldEventCounts = function(v){
return new Promise((resolve,reject) => {
if(!v.d.event_days||v.d.event_days==''){v.d.event_days=10}else{v.d.event_days=parseFloat(v.d.event_days)};
if(config.cron.deleteEvents===true&&v.d.event_days!==0){
knexQuery({
@ -323,18 +389,20 @@ const deleteOldEventCounts = function(v,callback){
['time','<', sqlDate(v.d.event_days+' DAY')],
]
},(err,rrr) => {
callback()
resolve()
if(err && err.code !== 'ER_NO_SUCH_TABLE')return console.error(err);
if(rrr.affectedRows && rrr.affectedRows.length > 0 || config.debugLog === true){
postMessage({f:'deleteEvents',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.event_days+' days deleted',ke:v.ke,time:moment()})
}
})
}else{
callback()
resolve()
}
})
}
//check for temporary files (special archive)
const deleteOldFileBins = function(v,callback){
const deleteOldFileBins = function(v){
return new Promise((resolve,reject) => {
if(!v.d.fileBin_days||v.d.fileBin_days==''){v.d.fileBin_days=10}else{v.d.fileBin_days=parseFloat(v.d.fileBin_days)};
if(config.cron.deleteFileBins===true&&v.d.fileBin_days!==0){
var fileBinQuery = " FROM Files WHERE ke=? AND `time` < ?";
@ -361,105 +429,50 @@ const deleteOldFileBins = function(v,callback){
})
}
}
callback()
resolve()
})
}else{
callback()
resolve()
}
}
//check for files with no database row
const checkForOrphanedFiles = function(v,callback){
if(config.cron.deleteOrphans === true){
console.log('"config.cron.deleteOrphans" has been removed. It has been replace by a one-time-run at startup with "config.insertOrphans". As the variable name suggests, instead of deleting, it will insert videos found without a database row.')
console.log('By default "config.orphanedVideoCheckMax" will only check up to 20 video. You can raise this value to any number you choose but be careful as it will check that number of videos on every start.')
}
callback()
})
}
//user processing function
const processUser = function(number,rows){
const processUser = async function(number,rows){
var v = rows[number];
if(!v){
//no user object given
return
}
s.debugLog(v)
s.debugLog(`Checking Group Key : ${v.ke}`)
s.debugLog(`Owner : ${v.mail}`)
if(!alreadyDeletedRowsWithNoVideosOnStart[v.ke]){
alreadyDeletedRowsWithNoVideosOnStart[v.ke]=false;
}
if(!overlapLocks[v.ke]){
// set overlap lock
overlapLocks[v.ke] = true
//set permissions
v.d=JSON.parse(v.details);
//size
if(!v.d.size||v.d.size==''){v.d.size=10000}else{v.d.size=parseFloat(v.d.size)};
//days to keep videos
if(!v.d.days||v.d.days==''){v.d.days=5}else{v.d.days=parseFloat(v.d.days)};
knexQuery({
action: "select",
columns: "*",
table: "Monitors",
where: [
['ke','=',v.ke],
]
},(err,rr) => {
if(!v.d.filters||v.d.filters==''){
v.d.filters={};
}
v.monitorsWithMaxKeepDays = []
rr.forEach(function(b,m){
b.details=JSON.parse(b.details);
if(b.details.max_keep_days&&b.details.max_keep_days!==''){
v.monitorsWithMaxKeepDays.push(b.mid)
v.d.filters['deleteOldVideosByCron'+b.mid]={
"id":'deleteOldVideosByCron'+b.mid,
"name":'deleteOldVideosByCron'+b.mid,
"sort_by":"time",
"sort_by_direction":"ASC",
"limit":"",
"enabled":"1",
"archive":"0",
"email":"0",
"delete":"1",
"execute":"",
"where":[{
"p1":"mid",
"p2":"=",
"p3":b.mid
},{
"p1":"end",
"p2":"<",
"p3": sqlDate(b.details.max_keep_days+" DAY")
}]
};
}
})
deleteOldLogs(v,function(){
await deleteOldVideos(v)
s.debugLog('--- deleteOldVideos Complete')
await deleteOldLogs(v)
s.debugLog('--- deleteOldLogs Complete')
deleteOldFileBins(v,function(){
await deleteOldFileBins(v)
s.debugLog('--- deleteOldFileBins Complete')
deleteOldEvents(v,function(){
await deleteOldEvents(v)
s.debugLog('--- deleteOldEvents Complete')
deleteOldEventCounts(v,function(){
await deleteOldEventCounts(v)
s.debugLog('--- deleteOldEventCounts Complete')
checkFilterRules(v,function(){
await checkFilterRules(v)
s.debugLog('--- checkFilterRules Complete')
deleteRowsWithNoVideo(v,function(){
await deleteRowsWithNoVideo(v)
s.debugLog('--- deleteRowsWithNoVideo Complete')
checkForOrphanedFiles(v,function(){
//done user, unlock current, and do next
overlapLocks[v.ke]=false;
processUser(number+1,rows)
})
})
})
})
})
})
})
})
await processUser(number+1,rows)
}else{
processUser(number+1,rows)
await processUser(number+1,rows)
}
}
//recursive function

12
libs/sql/utils.js
View File

@ -236,6 +236,17 @@ module.exports = (s,config,databaseOptions) => {
knexError(dbQuery,options,err)
}
}
const knexQueryPromise = (options) => {
return new Promise((resolve,reject) => {
knexQuery(options,(err,rows) => {
resolve({
ok: !err,
err: err,
rows: rows,
})
})
})
}
const initiateDatabaseEngine = () => {
s.databaseEngine = knex(databaseOptions)
return s.databaseEngine
@ -251,6 +262,7 @@ module.exports = (s,config,databaseOptions) => {
processWhereCondition: processWhereCondition,
knexError: knexError,
knexQuery: knexQuery,
knexQueryPromise: knexQueryPromise,
initiateDatabaseEngine: initiateDatabaseEngine
}
}

2
libs/video/utils.js
View File

@ -59,7 +59,7 @@ module.exports = (s,config,lang) => {
// const findCmd = [videosDirectory].concat(options.flags || ['-maxdepth','1'])
fs.writeFileSync(
tempDirectory + 'orphanCheck.sh',
`find "${videosDirectory}" -maxdepth 1 -type f -exec stat -c "%y %n" {} + | sort -r | head -n ${options.checkMax}`
`find "${videosDirectory}" -maxdepth 1 -type f -exec stat -c "%n" {} + | sort -r | head -n ${options.checkMax}`
);
let listing = spawn('sh',[tempDirectory + 'orphanCheck.sh'])
// const onData = options.onData ? options.onData : () => {}