From 811b6fc84600cd8c159a7f0904300da6b2d0e83b Mon Sep 17 00:00:00 2001 From: Moe Date: Thu, 6 Aug 2020 21:33:58 -0700 Subject: [PATCH] Modify Max Purge for Video, Timelapse Frames, FileBin, addStorage --- libs/auth.js | 2 +- libs/childNode.js | 2 +- libs/monitor.js | 13 +- libs/socketio.js | 2 +- libs/timelapse.js | 2 +- libs/user.js | 757 ++++++++++++++++++++------------------ libs/videoDropInServer.js | 2 +- libs/videos.js | 14 +- libs/webServerPaths.js | 25 +- package.json | 4 +- 10 files changed, 431 insertions(+), 392 deletions(-) diff --git a/libs/auth.js b/libs/auth.js index edd8822c..a3757a5e 100644 --- a/libs/auth.js +++ b/libs/auth.js @@ -263,7 +263,7 @@ module.exports = function(s,config,lang){ if(params.users === true){ s.knexQuery({ action: "select", - columns: columns, + columns: "*", table: "Users", where: [ ['details','NOT LIKE','%"sub"%'], diff --git a/libs/childNode.js b/libs/childNode.js index c8137731..dd6350ba 100644 --- a/libs/childNode.js +++ b/libs/childNode.js @@ -159,7 +159,7 @@ module.exports = function(s,config,lang,app,io){ //purge over max s.purgeDiskForGroup(d) //send new diskUsage values - s.setDiskUsedForGroup(d,insert.filesizeMB) + s.setDiskUsedForGroup(d.ke,insert.filesizeMB) clearTimeout(s.group[d.ke].activeMonitors[d.mid].recordingChecker) clearTimeout(s.group[d.ke].activeMonitors[d.mid].streamChecker) break; diff --git a/libs/monitor.js b/libs/monitor.js index 02c42717..825f10f6 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -104,6 +104,12 @@ module.exports = function(s,config,lang){ }); return x.ar; } + s.getFileBinDirectory = (monitor) => { + return s.dir.fileBin + monitor.ke + '/' + monitor.mid + '/' + } + s.getStreamsDirectory = (monitor) => { + return s.dir.streams + monitor.ke + '/' + monitor.mid + '/' + } s.getRawSnapshotFromMonitor = function(monitor,options,callback){ if(!callback){ callback = options @@ -539,9 +545,6 @@ module.exports = function(s,config,lang){ s.tx({f:'monitor_snapshot',snapshot:e.mon.name,snapshot_format:'plc',mid:e.mid,ke:e.ke},'GRP_'+e.ke) } } - s.getStreamsDirectory = (monitor) => { - return s.dir.streams + monitor.ke + '/' + monitor.mid + '/' - } const createRecordingDirectory = function(e,callback){ var directory if(e.details && e.details.dir && e.details.dir !== '' && config.childNodes.mode !== 'child'){ @@ -1658,11 +1661,9 @@ module.exports = function(s,config,lang){ var sqlQuery = 'SELECT * FROM Monitors WHERE ke=? AND ' var monitorQuery = [] var monitorPresets = {} - var didOne = false; preset.details.monitors.forEach(function(monitor){ const whereConditions = {} - if(!didOne){ - didOne = true + if(monitorQuery.length === 0){ whereConditions.ke = groupKey monitorQuery.push(['ke','=',groupKey]) }else{ diff --git a/libs/socketio.js b/libs/socketio.js index bdd7fed4..d1d3a16e 100644 --- a/libs/socketio.js +++ b/libs/socketio.js @@ -353,7 +353,7 @@ module.exports = function(s,config,lang,io){ } tx({f:'users_online',users:s.group[d.ke].users}) s.tx({f:'user_status_change',ke:d.ke,uid:cn.uid,status:1,user:s.group[d.ke].users[d.auth]},'GRP_'+d.ke) - s.sendDiskUsedAmountToClients(d) + s.sendDiskUsedAmountToClients(d.ke) s.loadGroupApps(d) s.knexQuery({ action: "select", diff --git a/libs/timelapse.js b/libs/timelapse.js index 56b3b206..e24c0f5f 100644 --- a/libs/timelapse.js +++ b/libs/timelapse.js @@ -79,7 +79,7 @@ module.exports = function(s,config,lang,app,io){ table: "Timelapse Frames", insert: queryInfo }) - s.setDiskUsedForGroup(e,queryInfo.size / 1048576,'timelapeFrames') + s.setDiskUsedForGroup(e.ke,queryInfo.size / 1048576,'timelapeFrames') s.purgeDiskForGroup(e) s.onInsertTimelapseFrameExtensions.forEach(function(extender){ extender(e,queryInfo,filePath) diff --git a/libs/user.js b/libs/user.js index dabf986a..8a3ec658 100644 --- a/libs/user.js +++ b/libs/user.js @@ -2,373 +2,410 @@ var fs = require('fs'); var events = require('events'); var spawn = require('child_process').spawn; var exec = require('child_process').exec; +var async = require("async"); module.exports = function(s,config,lang){ - s.purgeDiskForGroup = function(e){ - if(config.cron.deleteOverMax === true && s.group[e.ke] && s.group[e.ke].sizePurgeQueue){ - s.group[e.ke].sizePurgeQueue.push(1) - if(s.group[e.ke].sizePurging !== true){ - s.group[e.ke].sizePurging = true - var finish = function(){ - //remove value just used from queue - s.group[e.ke].sizePurgeQueue.shift() - //do next one - if(s.group[e.ke].sizePurgeQueue.length > 0){ - checkQueue() - }else{ - s.group[e.ke].sizePurging = false - s.sendDiskUsedAmountToClients(e) - } + const deleteSetOfVideos = function(options,callback){ + const groupKey = options.groupKey + const err = options.err + const videos = options.videos + const storageIndex = options.storageIndex + const reRunCheck = options.reRunCheck + var completedCheck = 0 + var whereGroup = [] + var whereQuery = [ + ['ke','=',groupKey], + ] + if(videos){ + var didOne = false + videos.forEach(function(video){ + video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext + if(didOne){ + whereGroup.push(['or','mid','=',video.mid]) + }else{ + didOne = true + whereGroup.push(['mid','=',video.mid]) } - var checkQueue = function(){ - //get first in queue - var currentPurge = s.group[e.ke].sizePurgeQueue[0] - var reRunCheck = function(){} - var deleteSetOfVideos = function(err,videos,storageIndex,callback){ - var completedCheck = 0 - var whereGroup = [] - var whereQuery = [ - ['ke','=',e.ke], - ] - if(videos){ - var didOne = false - videos.forEach(function(video){ - video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext - if(didOne){ - whereGroup.push( - ['or','mid','=',video.mid], - ['time','=',video.time] - ) - }else{ - didOne = true - whereGroup.push( - ['mid','=',video.mid], - ['time','=',video.time] - ) - } - fs.chmod(video.dir,0o777,function(err){ - fs.unlink(video.dir,function(err){ - ++completedCheck - if(err){ - fs.stat(video.dir,function(err){ - if(!err){ - s.file('delete',video.dir) - } - }) - } - const whereGroupLength = whereGroup.length / 2 - if(whereGroupLength > 0 && whereGroupLength === completedCheck){ - whereQuery[1] = whereGroup - s.knexQuery({ - action: "delete", - table: "Videos", - where: whereQuery - },(err) => { - reRunCheck() - }) - } - }) - }) - if(storageIndex){ - s.setDiskUsedForGroupAddStorage(e,{ - size: -(video.size/1048576), - storageIndex: storageIndex - }) - }else{ - s.setDiskUsedForGroup(e,-(video.size/1048576)) - } - s.tx({ - f: 'video_delete', - ff: 'over_max', - filename: s.formattedTime(video.time)+'.'+video.ext, - mid: video.mid, - ke: video.ke, - time: video.time, - end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss') - },'GRP_'+e.ke) - }) - }else{ - console.log(err) - } - if(whereGroup.length === 0){ - if(callback)callback() - } - } - var deleteSetOfTimelapseFrames = function(err,frames,storageIndex,callback){ - var whereGroup = [] - var whereQuery = [ - ['ke','=',e.ke], - [] - ] - var completedCheck = 0 - if(frames){ - var didOne = false - frames.forEach(function(frame){ - var selectedDate = frame.filename.split('T')[0] - var dir = s.getTimelapseFrameDirectory(frame) - var fileLocationMid = `${dir}` + frame.filename - if(didOne){ - whereGroup.push( - ['or','mid','=',frame.mid], - ['time','=',frame.time] - ) - }else{ - didOne = true - whereGroup.push( - ['mid','=',frame.mid], - ['time','=',frame.time] - ) - } - fs.unlink(fileLocationMid,function(err){ - ++completedCheck - if(err){ - fs.stat(fileLocationMid,function(err){ - if(!err){ - s.file('delete',fileLocationMid) - } - }) - } - const whereGroupLength = whereGroup.length / 2 - if(whereGroupLength > 0 && whereGroupLength === completedCheck){ - whereQuery[1] = whereGroup - s.knexQuery({ - action: "delete", - table: "Timelapse Frames", - where: whereQuery - },() => { - reRunCheck() - }) - } - }) - if(storageIndex){ - s.setDiskUsedForGroupAddStorage(e,{ - size: -(frame.size/1048576), - storageIndex: storageIndex - },'timelapeFrames') - }else{ - s.setDiskUsedForGroup(e,-(frame.size/1048576),'timelapeFrames') - } - // s.tx({ - // f: 'timelapse_frame_delete', - // ff: 'over_max', - // filename: s.formattedTime(video.time)+'.'+video.ext, - // mid: video.mid, - // ke: video.ke, - // time: video.time, - // end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss') - // },'GRP_'+e.ke) - }) - }else{ - console.log(err) - } - if(whereGroup.length === 0){ - if(callback)callback() - } - } - var deleteSetOfFileBinFiles = function(err,files,storageIndex,callback){ - var whereGroup = [] - var whereQuery = [ - ['ke','=',e.ke], - [] - ] - var completedCheck = 0 - if(files){ - files.forEach(function(file){ - var dir = s.getFileBinDirectory(file) - var fileLocationMid = `${dir}` + file.name - if(didOne){ - whereGroup.push( - ['or','mid','=',file.mid], - ['name','=',file.name] - ) - }else{ - didOne = true - whereGroup.push( - ['mid','=',file.mid], - ['name','=',file.name] - ) - } - fs.unlink(fileLocationMid,function(err){ - ++completedCheck - if(err){ - fs.stat(fileLocationMid,function(err){ - if(!err){ - s.file('delete',fileLocationMid) - } - }) - } - const whereGroupLength = whereGroup.length / 2 - if(whereGroupLength > 0 && whereGroupLength === completedCheck){ - whereQuery[1] = whereGroup - s.knexQuery({ - action: "delete", - table: "Files", - where: whereQuery - },() => { - reRunCheck() - }) - } - }) - if(storageIndex){ - s.setDiskUsedForGroupAddStorage(e,{ - size: -(file.size/1048576), - storageIndex: storageIndex - },'fileBin') - }else{ - s.setDiskUsedForGroup(e,-(file.size/1048576),'fileBin') + whereGroup.push(['time','=',video.time]) + fs.chmod(video.dir,0o777,function(err){ + fs.unlink(video.dir,function(err){ + ++completedCheck + if(err){ + fs.stat(video.dir,function(err){ + if(!err){ + s.file('delete',video.dir) } }) - }else{ - console.log(err) } - if(whereGroup.length === 0){ - if(callback)callback() - } - } - var deleteMainVideos = function(callback){ - reRunCheck = function(){ - return deleteMainVideos(callback) - } - //run purge command - if(s.group[e.ke].usedSpaceVideos > (s.group[e.ke].sizeLimit * (s.group[e.ke].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)){ + const whereGroupLength = whereGroup.length / 2 + if(whereGroupLength > 0 && whereGroupLength === completedCheck){ + whereQuery[1] = whereGroup s.knexQuery({ - action: "select", - columns: "*", + action: "delete", table: "Videos", - where: [ - ['ke','=',e.ke], - ['status','!=','0'], - ['details','NOT LIKE',`%"archived":"1"%`], - ['details','NOT LIKE',`%"dir"%`], - ], - orderBy: ['time','asc'], - limit: 3 - },(err,rows) => { - deleteSetOfVideos(err,rows,null,callback) - }) - }else{ - callback() - } - } - var deleteAddStorageVideos = function(callback){ - reRunCheck = function(){ - return deleteAddStorageVideos(callback) - } - var currentStorageNumber = 0 - var readStorageArray = function(finishedReading){ - setTimeout(function(){ - reRunCheck = readStorageArray - var storage = s.listOfStorage[currentStorageNumber] - if(!storage){ - //done all checks, move on to next user - callback() - return - } - var storageId = storage.value - if(storageId === '' || !s.group[e.ke].addStorageUse[storageId]){ - ++currentStorageNumber - readStorageArray() - return - } - var storageIndex = s.group[e.ke].addStorageUse[storageId] - //run purge command - if(storageIndex.usedSpace > (storageIndex.sizeLimit * (storageIndex.deleteOffset || config.cron.deleteOverMaxOffset))){ - s.knexQuery({ - action: "select", - columns: "*", - table: "Videos", - where: [ - ['ke','=',e.ke], - ['status','!=','0'], - ['details','NOT LIKE',`%"archived":"1"%`], - ['details','LIKE',`%"dir":"${storage.value}"%`], - ], - orderBy: ['time','asc'], - limit: 3 - },(err,rows) => { - deleteSetOfVideos(err,rows,storageIndex,callback) - }) - }else{ - ++currentStorageNumber - readStorageArray() - } + where: whereQuery + },(err,info) => { + reRunCheck() }) } - readStorageArray() - } - var deleteTimelapseFrames = function(callback){ - reRunCheck = function(){ - return deleteTimelapseFrames(callback) - } - //run purge command - if(s.group[e.ke].usedSpaceTimelapseFrames > (s.group[e.ke].sizeLimit * (s.group[e.ke].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){ - s.knexQuery({ - action: "select", - columns: "*", - table: "Timelapse Frames", - where: [ - ['ke','=',e.ke], - ['details','NOT LIKE',`%"archived":"1"%`], - ], - orderBy: ['time','asc'], - limit: 3 - },(err,frames) => { - deleteSetOfTimelapseFrames(err,frames,null,callback) - }) - }else{ - callback() - } - } - var deleteFileBinFiles = function(callback){ - if(config.deleteFileBinsOverMax === true){ - reRunCheck = function(){ - return deleteSetOfFileBinFiles(callback) + }) + }) + if(storageIndex){ + s.setDiskUsedForGroupAddStorage(groupKey,{ + size: -(video.size/1048576), + storageIndex: storageIndex + }) + }else{ + s.setDiskUsedForGroup(groupKey,-(video.size/1048576)) + } + s.tx({ + f: 'video_delete', + ff: 'over_max', + filename: s.formattedTime(video.time)+'.'+video.ext, + mid: video.mid, + ke: video.ke, + time: video.time, + end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss') + },'GRP_'+groupKey) + }) + }else{ + console.log(err) + } + if(whereGroup.length === 0){ + if(callback)callback() + } + } + var deleteSetOfTimelapseFrames = function(options,callback){ + const groupKey = options.groupKey + const err = options.err + const frames = options.frames + const storageIndex = options.storageIndex + var whereGroup = [] + var whereQuery = [ + ['ke','=',groupKey], + [] + ] + var completedCheck = 0 + if(frames){ + var didOne = false + frames.forEach(function(frame){ + var selectedDate = frame.filename.split('T')[0] + var dir = s.getTimelapseFrameDirectory(frame) + var fileLocationMid = `${dir}` + frame.filename + if(didOne){ + whereGroup.push( + ['or','mid','=',frame.mid], + ['time','=',frame.time] + ) + }else{ + didOne = true + whereGroup.push( + ['mid','=',frame.mid], + ['time','=',frame.time] + ) + } + fs.unlink(fileLocationMid,function(err){ + ++completedCheck + if(err){ + fs.stat(fileLocationMid,function(err){ + if(!err){ + s.file('delete',fileLocationMid) } - //run purge command - if(s.group[e.ke].usedSpaceFileBin > (s.group[e.ke].sizeLimit * (s.group[e.ke].sizeLimitFileBinPercent / 100) * config.cron.deleteOverMaxOffset)){ - s.knexQuery({ - action: "select", - columns: "*", - table: "Files", - where: [ - ['ke','=',e.ke], - ], - orderBy: ['time','asc'], - limit: 1 - },(err,frames) => { - deleteSetOfFileBinFiles(err,frames,null,callback) - }) - }else{ - callback() - } - }else{ - callback() - } + }) } - deleteMainVideos(function(){ - deleteTimelapseFrames(function(){ - deleteFileBinFiles(function(){ - deleteAddStorageVideos(function(){ - finish() + const whereGroupLength = whereGroup.length / 2 + if(whereGroupLength > 0 && whereGroupLength === completedCheck){ + whereQuery[1] = whereGroup + s.knexQuery({ + action: "delete", + table: "Timelapse Frames", + where: whereQuery + },() => { + deleteTimelapseFrames(groupKey,callback) + }) + } + }) + if(storageIndex){ + s.setDiskUsedForGroupAddStorage(groupKey,{ + size: -(frame.size/1048576), + storageIndex: storageIndex + },'timelapeFrames') + }else{ + s.setDiskUsedForGroup(groupKey,-(frame.size/1048576),'timelapeFrames') + } + // s.tx({ + // f: 'timelapse_frame_delete', + // ff: 'over_max', + // filename: s.formattedTime(video.time)+'.'+video.ext, + // mid: video.mid, + // ke: video.ke, + // time: video.time, + // end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss') + // },'GRP_'+groupKey) + }) + }else{ + console.log(err) + } + if(whereGroup.length === 0){ + if(callback)callback() + } + } + var deleteSetOfFileBinFiles = function(options,callback){ + const groupKey = options.groupKey + const err = options.err + const frames = options.frames + const storageIndex = options.storageIndex + var whereGroup = [] + var whereQuery = [ + ['ke','=',groupKey], + [] + ] + var completedCheck = 0 + if(files){ + files.forEach(function(file){ + var dir = s.getFileBinDirectory(file) + var fileLocationMid = `${dir}` + file.name + if(whereGroup.length !== 0){ + whereGroup.push( + ['or','mid','=',file.mid], + ['name','=',file.name] + ) + }else{ + whereGroup.push( + ['mid','=',file.mid], + ['name','=',file.name] + ) + } + fs.unlink(fileLocationMid,function(err){ + ++completedCheck + if(err){ + fs.stat(fileLocationMid,function(err){ + if(!err){ + s.file('delete',fileLocationMid) + } + }) + } + const whereGroupLength = whereGroup.length / 2 + if(whereGroupLength > 0 && whereGroupLength === completedCheck){ + whereQuery[1] = whereGroup + s.knexQuery({ + action: "delete", + table: "Files", + where: whereQuery + },() => { + deleteFileBinFiles(groupKey,callback) + }) + } + }) + if(storageIndex){ + s.setDiskUsedForGroupAddStorage(groupKey,{ + size: -(file.size/1048576), + storageIndex: storageIndex + },'fileBin') + }else{ + s.setDiskUsedForGroup(groupKey,-(file.size/1048576),'fileBin') + } + }) + }else{ + console.log(err) + } + if(whereGroup.length === 0){ + if(callback)callback() + } + } + var deleteAddStorageVideos = function(groupKey,callback){ + reRunCheck = function(){ + s.debugLog('deleteAddStorageVideos') + return deleteAddStorageVideos(groupKey,callback) + } + var currentStorageNumber = 0 + var readStorageArray = function(){ + setTimeout(function(){ + reRunCheck = readStorageArray + var storage = s.listOfStorage[currentStorageNumber] + if(!storage){ + //done all checks, move on to next user + callback() + return + } + var storageId = storage.value + if(storageId === '' || !s.group[groupKey].addStorageUse[storageId]){ + ++currentStorageNumber + readStorageArray() + return + } + var storageIndex = s.group[groupKey].addStorageUse[storageId] + //run purge command + if(storageIndex.usedSpace > (storageIndex.sizeLimit * (storageIndex.deleteOffset || config.cron.deleteOverMaxOffset))){ + s.knexQuery({ + action: "select", + columns: "*", + table: "Videos", + where: [ + ['ke','=',groupKey], + ['status','!=','0'], + ['details','NOT LIKE',`%"archived":"1"%`], + ['details','LIKE',`%"dir":"${storage.value}"%`], + ], + orderBy: ['time','asc'], + limit: 3 + },(err,rows) => { + deleteSetOfVideos({ + groupKey: groupKey, + err: err, + videos: rows, + storageIndex: storageIndex, + reRunCheck: () => { + return readStorageArray() + } + },callback) + }) + }else{ + ++currentStorageNumber + readStorageArray() + } + }) + } + readStorageArray() + } + var deleteMainVideos = function(groupKey,callback){ + // //run purge command + // s.debugLog('!!!!!!!!!!!deleteMainVideos') + // s.debugLog('s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)') + // s.debugLog(s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)) + // s.debugLog('s.group[groupKey].usedSpaceVideos') + // s.debugLog(s.group[groupKey].usedSpaceVideos) + // s.debugLog('s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset') + // s.debugLog(s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset) + // s.debugLog('s.group[groupKey].sizeLimitVideoPercent / 100') + // s.debugLog(s.group[groupKey].sizeLimitVideoPercent / 100) + // s.debugLog('s.group[groupKey].sizeLimit') + // s.debugLog(s.group[groupKey].sizeLimit) + if(s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)){ + s.knexQuery({ + action: "select", + columns: "*", + table: "Videos", + where: [ + ['ke','=',groupKey], + ['status','!=','0'], + ['details','NOT LIKE',`%"archived":"1"%`], + ['details','NOT LIKE',`%"dir"%`], + ], + orderBy: ['time','asc'], + limit: 3 + },(err,rows) => { + deleteSetOfVideos({ + groupKey: groupKey, + err: err, + videos: rows, + storageIndex: null, + reRunCheck: () => { + return deleteMainVideos(groupKey,callback) + } + },callback) + }) + }else{ + callback() + } + } + var deleteTimelapseFrames = function(groupKey,callback){ + //run purge command + if(s.group[groupKey].usedSpaceTimelapseFrames > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){ + s.knexQuery({ + action: "select", + columns: "*", + table: "Timelapse Frames", + where: [ + ['ke','=',groupKey], + ['details','NOT LIKE',`%"archived":"1"%`], + ], + orderBy: ['time','asc'], + limit: 3 + },(err,frames) => { + deleteSetOfTimelapseFrames({ + groupKey: groupKey, + err: err, + frames: frames, + storageIndex: null + },callback) + }) + }else{ + callback() + } + } + var deleteFileBinFiles = function(groupKey,callback){ + if(config.deleteFileBinsOverMax === true){ + //run purge command + if(s.group[groupKey].usedSpaceFileBin > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitFileBinPercent / 100) * config.cron.deleteOverMaxOffset)){ + s.knexQuery({ + action: "select", + columns: "*", + table: "Files", + where: [ + ['ke','=',groupKey], + ], + orderBy: ['time','asc'], + limit: 1 + },(err,frames) => { + deleteSetOfFileBinFiles({ + groupKey: groupKey, + err: err, + frames: frames, + storageIndex: null + },callback) + }) + }else{ + callback() + } + }else{ + callback() + } + } + let purgeDiskGroup = () => {} + const runQuery = async.queue(function(e, callback) { + purgeDiskGroup(e,callback) + }, 1); + if(config.cron.deleteOverMax === true){ + purgeDiskGroup = (e,callback) => { + const groupKey = e.ke + if(s.group[groupKey]){ + if(s.group[groupKey].sizePurging !== true){ + s.group[groupKey].sizePurging = true + s.debugLog(`${groupKey} deleteMainVideos`) + deleteMainVideos(groupKey,() => { + s.debugLog(`${groupKey} deleteTimelapseFrames`) + deleteTimelapseFrames(groupKey,() => { + s.debugLog(`${groupKey} deleteFileBinFiles`) + deleteFileBinFiles(groupKey,() => { + s.debugLog(`${groupKey} deleteAddStorageVideos`) + deleteAddStorageVideos(groupKey,() => { + s.group[groupKey].sizePurging = false + s.sendDiskUsedAmountToClients(groupKey) + callback(); }) }) }) }) + }else{ + s.sendDiskUsedAmountToClients(groupKey) } - checkQueue() } - }else{ - s.sendDiskUsedAmountToClients(e) } } - s.setDiskUsedForGroup = function(e,bytes,storagePoint){ + s.purgeDiskForGroup = (e) => { + return runQuery.push(e,function(){ + //... + }) + } + s.setDiskUsedForGroup = function(groupKey,bytes,storagePoint){ //`bytes` will be used as the value to add or substract - if(s.group[e.ke] && s.group[e.ke].diskUsedEmitter){ - s.group[e.ke].diskUsedEmitter.emit('set',bytes,storagePoint) + if(s.group[groupKey] && s.group[groupKey].diskUsedEmitter){ + s.group[groupKey].diskUsedEmitter.emit('set',bytes,storagePoint) } } - s.setDiskUsedForGroupAddStorage = function(e,data,storagePoint){ - if(s.group[e.ke] && s.group[e.ke].diskUsedEmitter){ - s.group[e.ke].diskUsedEmitter.emit('setAddStorage',data,storagePoint) + s.setDiskUsedForGroupAddStorage = function(groupKey,data,storagePoint){ + if(s.group[groupKey] && s.group[groupKey].diskUsedEmitter){ + s.group[groupKey].diskUsedEmitter.emit('setAddStorage',data,storagePoint) } } s.purgeCloudDiskForGroup = function(e,storageType,storagePoint){ @@ -382,19 +419,19 @@ module.exports = function(s,config,lang){ s.group[e.ke].diskUsedEmitter.emit('setCloud',usage,storagePoint) } } - s.sendDiskUsedAmountToClients = function(e){ + s.sendDiskUsedAmountToClients = function(groupKey){ //send the amount used disk space to connected users - if(s.group[e.ke]&&s.group[e.ke].init){ + if(s.group[groupKey]&&s.group[groupKey].init){ s.tx({ f: 'diskUsed', - size: s.group[e.ke].usedSpace, - usedSpace: s.group[e.ke].usedSpace, - usedSpaceVideos: s.group[e.ke].usedSpaceVideos, - usedSpaceFilebin: s.group[e.ke].usedSpaceFilebin, - usedSpaceTimelapseFrames: s.group[e.ke].usedSpaceTimelapseFrames, - limit: s.group[e.ke].sizeLimit, - addStorage: s.group[e.ke].addStorageUse - },'GRP_'+e.ke); + size: s.group[groupKey].usedSpace, + usedSpace: s.group[groupKey].usedSpace, + usedSpaceVideos: s.group[groupKey].usedSpaceVideos, + usedSpaceFilebin: s.group[groupKey].usedSpaceFilebin, + usedSpaceTimelapseFrames: s.group[groupKey].usedSpaceTimelapseFrames, + limit: s.group[groupKey].sizeLimit, + addStorage: s.group[groupKey].addStorageUse + },'GRP_'+groupKey); } } //user log @@ -442,7 +479,7 @@ module.exports = function(s,config,lang){ //save global used space as megabyte value s.group[e.ke].usedSpace = s.group[e.ke].usedSpace || ((e.size || 0) / 1048576) //emit the changes to connected users - s.sendDiskUsedAmountToClients(e) + s.sendDiskUsedAmountToClients(e.ke) } s.loadGroupApps = function(e){ // e = user @@ -501,7 +538,7 @@ module.exports = function(s,config,lang){ var cloudDisk = s.group[e.ke].cloudDiskUse[storageType] //set queue processor var finish=function(){ - // s.sendDiskUsedAmountToClients(e) + // s.sendDiskUsedAmountToClients(e.ke) } var deleteVideos = function(){ //run purge command @@ -624,7 +661,7 @@ module.exports = function(s,config,lang){ }) }) }else{ - // s.sendDiskUsedAmountToClients(e) + // s.sendDiskUsedAmountToClients(e.ke) } }) //s.setDiskUsedForGroup @@ -652,7 +689,7 @@ module.exports = function(s,config,lang){ break; } //remove value just used from queue - s.sendDiskUsedAmountToClients(e) + s.sendDiskUsedAmountToClients(e.ke) }) s.group[e.ke].diskUsedEmitter.on('setAddStorage',function(data,storageType){ var currentSize = data.size @@ -680,7 +717,7 @@ module.exports = function(s,config,lang){ break; } //remove value just used from queue - s.sendDiskUsedAmountToClients(e) + s.sendDiskUsedAmountToClients(e.ke) }) } Object.keys(details).forEach(function(v){ diff --git a/libs/videoDropInServer.js b/libs/videoDropInServer.js index 285427b9..744e758a 100644 --- a/libs/videoDropInServer.js +++ b/libs/videoDropInServer.js @@ -43,7 +43,7 @@ module.exports = function(s,config,lang,app,io){ time: timeNow, } }) - s.setDiskUsedForGroup(monitor,fileStats.size / 1048576) + s.setDiskUsedForGroup(monitor.ke,fileStats.size / 1048576) } // else{ // s.insertDatabaseRow( diff --git a/libs/videos.js b/libs/videos.js index 2c038bf2..4def84fd 100644 --- a/libs/videos.js +++ b/libs/videos.js @@ -176,12 +176,12 @@ module.exports = function(s,config,lang){ //send new diskUsage values var storageIndex = s.getVideoStorageIndex(e) if(storageIndex){ - s.setDiskUsedForGroupAddStorage(e,{ + s.setDiskUsedForGroupAddStorage(e.ke,{ size: k.filesizeMB, storageIndex: storageIndex }) }else{ - s.setDiskUsedForGroup(e,k.filesizeMB) + s.setDiskUsedForGroup(e.ke,k.filesizeMB) } s.onBeforeInsertCompletedVideoExtensions.forEach(function(extender){ extender(e,k) @@ -238,12 +238,12 @@ module.exports = function(s,config,lang){ },'GRP_'+e.ke); var storageIndex = s.getVideoStorageIndex(e) if(storageIndex){ - s.setDiskUsedForGroupAddStorage(e,{ + s.setDiskUsedForGroupAddStorage(e.ke,{ size: -(r.size / 1048576), storageIndex: storageIndex }) }else{ - s.setDiskUsedForGroup(e,-(r.size / 1048576)) + s.setDiskUsedForGroup(e.ke,-(r.size / 1048576)) } s.knexQuery({ action: "delete", @@ -303,12 +303,12 @@ module.exports = function(s,config,lang){ },'GRP_'+video.ke); var storageIndex = s.getVideoStorageIndex(video) if(storageIndex){ - s.setDiskUsedForGroupAddStorage(video,{ + s.setDiskUsedForGroupAddStorage(video.ke,{ size: -(video.size / 1048576), storageIndex: storageIndex }) }else{ - s.setDiskUsedForGroup(video,-(video.size / 1048576)) + s.setDiskUsedForGroup(video.ke,-(video.size / 1048576)) } fs.unlink(video.dir + filename,function(err){ fs.stat(video.dir + filename,function(err){ @@ -543,7 +543,7 @@ module.exports = function(s,config,lang){ time: timeNow, } }) - s.setDiskUsedForGroup({ke: ke},fileStats.size / 1048576,'fileBin') + s.setDiskUsedForGroup(ke,fileStats.size / 1048576,'fileBin') fs.unlink(commandTempLocation,function(){ }) diff --git a/libs/webServerPaths.js b/libs/webServerPaths.js index d4e576f6..ebab1362 100644 --- a/libs/webServerPaths.js +++ b/libs/webServerPaths.js @@ -154,6 +154,7 @@ module.exports = function(s,config,lang,app,io){ s.checkCorrectPathEnding(config.webPaths.admin)+':screen', s.checkCorrectPathEnding(config.webPaths.super)+':screen', ],function (req,res){ + var response = {ok: false}; req.ip = s.getClientIp(req) var screenChooser = function(screen){ var search = function(screen){ @@ -366,7 +367,7 @@ module.exports = function(s,config,lang,app,io){ chosenRender = r.details.landing_page } renderPage(config.renderPaths[chosenRender],{ - $user:response, + $user: response, config: s.getConfigWithBranding(req.hostname), lang:r.lang, define:s.getDefinitonFile(r.details.lang), @@ -392,7 +393,6 @@ module.exports = function(s,config,lang,app,io){ ], limit: 1 },(err,r) => { - var response = {ok: false}; if(!err && r && r[0]){ r=r[0];r.auth=s.md5(s.gid()); s.knexQuery({ @@ -406,7 +406,7 @@ module.exports = function(s,config,lang,app,io){ ['uid','=',r.uid], ] }) - var response = { + response = { ok: true, auth_token: r.auth, ke: r.ke, @@ -466,9 +466,9 @@ module.exports = function(s,config,lang,app,io){ },function(err,rr) { if(rr && rr[0]){ rr=rr[0]; - rr.details=JSON.parse(rr.details); - r.details.mon_groups=rr.details.mon_groups; - response.details=JSON.stringify(r.details); + rr.details = JSON.parse(rr.details); + r.details.mon_groups = rr.details.mon_groups; + response.details = JSON.stringify(r.details); factorAuth() }else{ failedAuthentication(req.body.function) @@ -540,7 +540,7 @@ module.exports = function(s,config,lang,app,io){ if(!user.uid){ user.uid=s.gid() } - var response = { + response = { ke:req.body.key, uid:user.uid, auth:s.createHash(s.gid()), @@ -565,10 +565,10 @@ module.exports = function(s,config,lang,app,io){ },function(err,rr) { if(rr&&rr[0]){ //already registered - rr=rr[0] - var response = rr; - rr.details=JSON.parse(rr.details) - response.lang=s.getLanguageFile(rr.details.lang) + rr = rr[0] + response = rr; + rr.details = JSON.parse(rr.details) + response.lang = s.getLanguageFile(rr.details.lang) s.userLog({ke:req.body.key,mid:'$USER'},{type:r.lang['LDAP User Authenticated'],msg:{user:user,shinobiUID:rr.uid}}) s.knexQuery({ action: "update", @@ -680,7 +680,6 @@ module.exports = function(s,config,lang,app,io){ } } req.body.function = s.factorAuth[req.body.ke][req.body.id].function - var response = s.factorAuth[req.body.ke][req.body.id].info checkRoute(s.factorAuth[req.body.ke][req.body.id].user) clearTimeout(s.factorAuth[req.body.ke][req.body.id].expireAuth) s.deleteFactorAuth({ @@ -1110,6 +1109,7 @@ module.exports = function(s,config,lang,app,io){ s.sqlQueryBetweenTimesWithPermissions({ table: videoSet, user: user, + noCount: true, groupKey: req.params.ke, monitorId: req.params.id, startTime: req.query.start, @@ -1683,6 +1683,7 @@ module.exports = function(s,config,lang,app,io){ s.sqlQueryBetweenTimesWithPermissions({ table: 'Events Counts', user: user, + noCount: true, groupKey: req.params.ke, monitorId: req.params.id, startTime: req.query.start, diff --git a/package.json b/package.json index fd714f4a..865fb374 100644 --- a/package.json +++ b/package.json @@ -26,11 +26,11 @@ "ftp-srv": "^4.3.1", "http-proxy": "^1.17.0", "jsonfile": "^3.0.1", - "knex": "^0.19.5", + "knex": "^0.21.2", "ldapauth-fork": "^4.0.2", "moment": "^2.17.0", "mp4frag": "^0.2.0", - "mysql": "^2.16.0", + "mysql": "^2.18.1", "node-onvif": "^0.1.7", "node-ssh": "^5.1.2", "nodemailer": "^4.0.1",