ShinobiSystems
/
Shinobi
mirror of https://gitlab.com/Shinobi-Systems/Shinobi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add FileBin Space Calculator 

+ make timelapse range queries have time as well
pushover
Moe 2019-07-26 08:18:24 -07:00
parent e474eba577
commit 2787883ea6
5 changed files with 87 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
languages/en_CA.json
View File

@ -235,6 +235,7 @@
"Use Max Storage Amount": "Use Max Storage Amount",
"Max Storage Amount": "Max Storage Amount",
"Video Share": "Video Share",
"FileBin Share": "FileBin Share",
"Timelapse Frames Share": "Timelapse Frames Share",
"Number of Days to keep": "Number of Days to keep",
"Monitor Groups": "Monitor Groups",

3
libs/config.js
View File

@ -27,7 +27,8 @@ module.exports = function(s){
if(config.cron.deleteOverMaxOffset === undefined)config.cron.deleteOverMaxOffset=0.9;
if(config.cron.deleteLogs === undefined)config.cron.deleteLogs=true;
if(config.cron.deleteEvents === undefined)config.cron.deleteEvents=true;
if(config.cron.deleteFileBins === undefined)config.cron.deleteFileBins=true;
if(config.cron.deleteFileBinsOverMax === undefined)config.cron.deleteFileBins=true;
if(config.deleteFileBins === undefined)config.deleteFileBinsOverMax=true;
if(config.cron.interval === undefined)config.cron.interval=1;
if(config.databaseType === undefined){config.databaseType='mysql'}
if(config.pluginKeys === undefined)config.pluginKeys={};

13
libs/fileBin.js Normal file
View File

@ -0,0 +1,13 @@
var fs = require('fs')
var moment = require('moment')
module.exports = function(s,config,lang,app,io){
s.getFileBinDirectory = function(e){
if(e.mid&&!e.id){e.id=e.mid}
s.checkDetails(e)
if(e.details&&e.details.dir&&e.details.dir!==''){
return s.checkCorrectPathEnding(e.details.dir)+e.ke+'/'+e.id+'/'
}else{
return s.dir.fileBin+e.ke+'/'+e.id+'/';
}
}
}

4
libs/timelapse.js
View File

@ -337,8 +337,8 @@ module.exports = function(s,config,lang,app,io){
var dateNow = new Date()
var hoursNow = dateNow.getHours()
if(hoursNow === 1){
var dateNowMoment = moment(dateNow).utc().format('YYYY-MM-DD')
var dateMinusOneDay = moment(dateNow).utc().subtract(1, 'days').format('YYYY-MM-DD')
var dateNowMoment = moment(dateNow).utc().format('YYYY-MM-DDTHH:mm:ss')
var dateMinusOneDay = moment(dateNow).utc().subtract(1, 'days').format('YYYY-MM-DDTHH:mm:ss')
s.sqlQuery('SELECT * FROM `Timelapse Frames` WHERE time => ? AND time =< ?',[dateMinusOneDay,dateNowMoment],function(err,frames){
console.log(frames.length)
var groups = {}

74
libs/user.js
View File

@ -83,7 +83,8 @@ module.exports = function(s,config){
if(frames){
frames.forEach(function(frame){
var selectedDate = frame.filename.split('T')[0]
var fileLocationMid = `${frame.ke}/${frame.mid}_timelapse/${selectedDate}/` + frame.filename
var dir = s.getTimelapseFrameDirectory(frame)
var fileLocationMid = `${dir}` + frame.filename
framesToDelete.push('(mid=? AND `time`=?)')
queryValues.push(frame.mid)
queryValues.push(frame.time)
@ -128,6 +129,49 @@ module.exports = function(s,config){
if(callback)callback()
}
}
var deleteSetOfFileBinFiles = function(err,files,storageIndex,callback){
var filesToDelete = []
var queryValues = [e.ke]
var completedCheck = 0
if(files){
files.forEach(function(file){
var dir = s.getFileBinDirectory(file)
var fileLocationMid = `${dir}` + file.name
filesToDelete.push('(mid=? AND `name`=?)')
queryValues.push(file.mid)
queryValues.push(file.name)
fs.unlink(fileLocationMid,function(err){
++completedCheck
if(err){
fs.stat(fileLocationMid,function(err){
if(!err){
s.file('delete',fileLocationMid)
}
})
}
if(filesToDelete.length === completedCheck){
filesToDelete = filesToDelete.join(' OR ')
s.sqlQuery('DELETE FROM `Files` WHERE ke =? AND ('+filesToDelete+')',queryValues,function(){
reRunCheck()
})
}
})
if(storageIndex){
s.setDiskUsedForGroupAddStorage(e,{
size: -(file.size/1000000),
storageIndex: storageIndex
},'fileBin')
}else{
s.setDiskUsedForGroup(e,-(file.size/1000000),'fileBin')
}
})
}else{
console.log(err)
}
if(framesToDelete.length === 0){
if(callback)callback()
}
}
var deleteMainVideos = function(callback){
reRunCheck = function(){
return deleteMainVideos(callback)
@ -188,10 +232,29 @@ module.exports = function(s,config){
callback()
}
}
var deleteFileBinFiles = function(callback){
if(config.deleteFileBinsOverMax === true){
reRunCheck = function(){
return deleteSetOfFileBinFiles(callback)
}
//run purge command
if(s.group[e.ke].usedSpaceFileBin > (s.group[e.ke].sizeLimit * (s.group[e.ke].sizeLimitFileBinPercent / 100) * config.cron.deleteOverMaxOffset)){
s.sqlQuery('SELECT * FROM `Files` WHERE ke=? ORDER BY `time` ASC LIMIT 1',[e.ke],function(err,frames){
deleteSetOfFileBinFiles(err,frames,null,callback)
})
}else{
callback()
}
}else{
callback()
}
}
deleteMainVideos(function(){
deleteTimelapseFrames(function(){
deleteAddStorageVideos(function(){
finish()
deleteFileBinFiles(function(){
deleteAddStorageVideos(function(){
finish()
})
})
})
})
@ -268,7 +331,8 @@ module.exports = function(s,config){
//save global space limit for group key (mb)
s.group[e.ke].sizeLimit = e.limit || s.group[e.ke].sizeLimit || 10000
s.group[e.ke].sizeLimitVideoPercent = parseFloat(s.group[e.ke].init.size_video_percent) || 90
s.group[e.ke].sizeLimitTimelapseFramesPercent = parseFloat(s.group[e.ke].init.size_timelapse_percent) || 10
s.group[e.ke].sizeLimitTimelapseFramesPercent = parseFloat(s.group[e.ke].init.size_timelapse_percent) || 5
s.group[e.ke].sizeLimitFileBinPercent = parseFloat(s.group[e.ke].init.size_filebin_percent) || 5
//save global used space as megabyte value
s.group[e.ke].usedSpace = s.group[e.ke].usedSpace || ((e.size || 0) / 1000000)
//emit the changes to connected users
@ -377,7 +441,7 @@ module.exports = function(s,config){
s.deleteVideoFromCloudExtensionsRunner(e,storageType,frame)
})
s.sqlQuery('DELETE FROM `Cloud Timelapse Frames` WHERE ke =? AND ('+framesToDelete+')',queryValues,function(){
deleteTimelapseFrames(callback)
})
})
}else{