ShinobiSystems
/
Shinobi
mirror of https://gitlab.com/Shinobi-Systems/Shinobi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compound Update 

- Calculate Max Storage Amount for Videos and Timelapse Frames separately
- Split Max Storage Amount by percentage for Videos and Timelapse Frames
- Fix order of operations for on start monitor load and orphan check
- move some extender definitions to extenders.js
- Slim Power Video Viewer event bars
- Add hours to date range selector for Timelapse Jpeg
merge-requests/61/head
Moe 2019-05-05 13:30:07 -07:00
parent cfa26f2f4e
commit a87c2e6acc
10 changed files with 200 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
definitions/en_CA.js
View File

@ -3525,12 +3525,10 @@ module.exports = function(s,config,lang){
"possible": [
{
"name": "No",
"value": "0",
"info": "No is the default."
},
{
"name": "Yes",
"value": "1",
"info": "Do this if you are having recurring issues only."
}
]
@ -3688,6 +3686,20 @@ module.exports = function(s,config,lang){
"notForSubAccount": true,
"evaluation": "details.edit_size !== '0'"
},
{
"name": "detail=size_video_percent",
"field": lang["Video Share"],
"description": "Percent of Max Storage Amount the videos can record to.",
"default": "60",
"notForSubAccount": true,
},
{
"name": "detail=size_timelapse_percent",
"field": lang["Timelapse Frames Share"],
"description": "Percent of Max Storage Amount the timelapse frames can record to.",
"default": "40",
"notForSubAccount": true,
},
{
hidden:true,
"name": "detail=addStorage",

2
languages/en_CA.json
View File

@ -220,6 +220,8 @@
"2-Factor Authentication": "2-Factor Authentication",
"Use Max Storage Amount": "Use Max Storage Amount",
"Max Storage Amount": "Max Storage Amount",
"Video Share": "Video Share",
"Timelapse Frames Share": "Timelapse Frames Share",
"Number of Days to keep": "Number of Days to keep",
"Monitor Groups": "Monitor Groups",
"Group Name": "Group Name",

10
libs/extenders.js
View File

@ -153,4 +153,14 @@ module.exports = function(s,config){
s.onGetRamUsageExtensions.push(callback)
}
//
/////// VIDEOS ////////
s.insertCompletedVideoExtensions = []
s.insertCompletedVideoExtender = function(callback){
s.insertCompletedVideoExtensions.push(callback)
}
/////// TIMELAPSE ////////
s.onInsertTimelapseFrameExtensions = []
s.onInsertTimelapseFrame = function(callback){
s.onInsertTimelapseFrameExtensions.push(callback)
}
}

15
libs/monitor.js
View File

@ -923,9 +923,20 @@ module.exports = function(s,config,lang){
details.dir = e.details.dir
}
var timeNow = new Date()
s.sqlQuery('INSERT INTO `Timelapse Frames` (ke,mid,details,filename,size,time) VALUES (?,?,?,?,?,?)',[e.ke,e.id,s.s(details),filename,fileStats.size,timeNow])
s.setDiskUsedForGroup(e,fileStats.size / 1000000)
var queryInfo = {
ke: e.ke,
mid: e.id,
details: s.s(details),
filename: filename,
size: fileStats.size,
time: timeNow
}
s.sqlQuery('INSERT INTO `Timelapse Frames` ('+Object.keys(queryInfo).join(',')+') VALUES (?,?,?,?,?,?)',Object.values(queryInfo))
s.setDiskUsedForGroup(e,fileStats.size / 1000000,'timelapeFrames')
s.purgeDiskForGroup(e)
s.onInsertTimelapseFrameExtensions.forEach(function(extender){
extender(e,queryInfo)
})
})
s.group[e.ke].mon[e.id].recordTimelapseWriter = fileStream
}

127
libs/startup.js
View File

@ -33,6 +33,7 @@ module.exports = function(s,config,lang,io){
}
}
var loadedAccounts = []
var foundMonitors = []
var loadMonitors = function(callback){
s.beforeMonitorsLoadedOnStartupExtensions.forEach(function(extender){
extender()
@ -40,6 +41,7 @@ module.exports = function(s,config,lang,io){
s.systemLog(lang.startUpText4)
//preliminary monitor start
s.sqlQuery('SELECT * FROM Monitors', function(err,monitors) {
foundMonitors = monitors
if(err){s.systemLog(err)}
if(monitors && monitors[0]){
var loadCompleted = 0
@ -52,18 +54,12 @@ module.exports = function(s,config,lang,io){
s.sendMonitorStatus({id:monitor.mid,ke:monitor.ke,status:'Stopped'});
var monObj = Object.assign(monitor,{id : monitor.mid})
s.camera(monitor.mode,monObj)
s.orphanedVideoCheck(monitor,2,function(orphanedFilesCount){
if(orphanedFilesCount){
orphanedVideosForMonitors[monitor.ke][monitor.mid] += orphanedFilesCount
}
++loadCompleted
if(monitors[loadCompleted]){
loadMonitor(monitors[loadCompleted])
}else{
s.systemLog(lang.startUpText6+' : '+s.s(orphanedVideosForMonitors))
callback()
}
})
++loadCompleted
if(monitors[loadCompleted]){
loadMonitor(monitors[loadCompleted])
}else{
callback()
}
}
loadMonitor(monitors[loadCompleted])
}else{
@ -71,36 +67,85 @@ module.exports = function(s,config,lang,io){
}
})
}
var checkForOrphanedVideos = function(callback){
var monitors = foundMonitors
if(monitors && monitors[0]){
var loadCompleted = 0
var orphanedVideosForMonitors = {}
var checkForOrphanedVideosForMonitor = function(monitor){
if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {}
if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0
s.orphanedVideoCheck(monitor,2,function(orphanedFilesCount){
if(orphanedFilesCount){
orphanedVideosForMonitors[monitor.ke][monitor.mid] += orphanedFilesCount
}
++loadCompleted
if(monitors[loadCompleted]){
checkForOrphanedVideosForMonitor(monitors[loadCompleted])
}else{
s.systemLog(lang.startUpText6+' : '+s.s(orphanedVideosForMonitors))
delete(foundMonitors)
callback()
}
})
}
checkForOrphanedVideosForMonitor(monitors[loadCompleted])
}else{
callback()
}
}
var loadDiskUseForUser = function(user,callback){
s.systemLog(user.mail+' : '+lang.startUpText0)
var userDetails = JSON.parse(user.details)
s.group[user.ke].sizeLimit = parseFloat(userDetails.size) || 10000
// s.group[user.ke].sizeLimitVideoPercent = parseFloat(userDetails.size_video_percent) || 60
// s.group[user.ke].sizeLimitTimelapseFramesPercent = parseFloat(userDetails.size_timelapse_percent) || 40
s.group[user.ke].sizeLimitVideoPercent = parseFloat(userDetails.size_video_percent) || 90
s.group[user.ke].sizeLimitTimelapseFramesPercent = parseFloat(userDetails.size_timelapse_percent) || 10
s.sqlQuery('SELECT * FROM Videos WHERE ke=? AND status!=?',[user.ke,0],function(err,videos){
s.sqlQuery('SELECT * FROM `Timelapse Frames` WHERE ke=?',[user.ke],function(err,timelapseFrames){
s.sqlQuery('SELECT * FROM `Files` WHERE ke=?',[user.ke],function(err,files){
var usedSpace = 0
var usedSpaceVideos = 0
var usedSpaceTimelapseFrames = 0
var usedSpaceFilebin = 0
var addStorageData = {
files: [],
videos: [],
timelapeFrames: [],
}
if(videos && videos[0]){
videos.forEach(function(video){
video.details = s.parseJSON(video.details)
if(!video.details.dir){
usedSpace += video.size
usedSpaceVideos += video.size
}else{
addStorageVideos.videos.push(video)
}
})
}
if(timelapseFrames && timelapseFrames[0]){
timelapseFrames.forEach(function(frame){
usedSpace += frame.size
frame.details = s.parseJSON(frame.details)
if(!frame.details.dir){
usedSpaceTimelapseFrames += frame.size
}else{
addStorageVideos.timelapeFrames.push(frame)
}
})
}
if(files && files[0]){
files.forEach(function(file){
usedSpace += file.size
file.details = s.parseJSON(file.details)
if(!file.details.dir){
usedSpaceFilebin += file.size
}else{
addStorageData.files.push(file)
}
})
}
s.group[user.ke].usedSpace = usedSpace / 1000000
loadAddStorageDiskUseForUser(user,videos,function(){
s.group[user.ke].usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1000000
s.group[user.ke].usedSpaceVideos = usedSpaceVideos / 1000000
s.group[user.ke].usedSpaceFilebin = usedSpaceFilebin / 1000000
s.group[user.ke].usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1000000
loadAddStorageDiskUseForUser(user,addStorageData,function(){
callback()
})
})
@ -137,7 +182,10 @@ module.exports = function(s,config,lang,io){
callback()
})
}
var loadAddStorageDiskUseForUser = function(user,videos,callback){
var loadAddStorageDiskUseForUser = function(user,data,callback){
var videos = data.videos
var timelapseFrames = data.timelapseFrames
var files = data.files
var userDetails = JSON.parse(user.details)
var userAddStorageData = s.parseJSON(userDetails.addStorage) || {}
var currentStorageNumber = 0
@ -162,14 +210,34 @@ module.exports = function(s,config,lang,io){
storageIndex.path = path
storageIndex.usedSpace = 0
storageIndex.sizeLimit = parseFloat(storageData.limit) || parseFloat(user.limit) || 10000
var usedSpaceVideos = 0
var usedSpaceTimelapseFrames = 0
var usedSpaceFilebin = 0
if(videos && videos[0]){
videos.forEach(function(video){
if(video.details.dir === storage.value){
storageIndex.usedSpace += video.size
usedSpaceVideos += video.size
}
})
storageIndex.usedSpace = storageIndex.usedSpace / 1000000
}
if(timelapseFrames && timelapseFrames[0]){
timelapseFrames.forEach(function(frame){
if(video.details.dir === storage.value){
usedSpaceTimelapseFrames += frame.size
}
})
}
if(files && files[0]){
files.forEach(function(file){
if(video.details.dir === storage.value){
usedSpaceFilebin += file.size
}
})
}
storageIndex.usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1000000
storageIndex.usedSpaceVideos = usedSpaceVideos / 1000000
storageIndex.usedSpaceFilebin = usedSpaceFilebin / 1000000
storageIndex.usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1000000
s.systemLog(user.mail+' : '+path+' : '+videos.length,storageIndex.usedSpace)
++currentStorageNumber
readStorageArray()
@ -242,11 +310,14 @@ module.exports = function(s,config,lang,io){
s.preQueries()
setTimeout(function(){
checkForTerminalCommands(function(){
//load administrators (groups)
loadAdminUsers(function(){
//load monitors (for groups)
loadMonitors(function(){
s.processReady()
//load monitors (for groups)
loadMonitors(function(){
//load administrators (groups)
loadAdminUsers(function(){
//check for orphaned videos
checkForOrphanedVideos(function(){
s.processReady()
})
})
})
})

2
libs/uploaders/loader.js
View File

@ -5,6 +5,7 @@ module.exports = function(s){
s.insertCompletedVideoExtender(opt.insertCompletedVideoExtender)
s.deleteVideoFromCloudExtensions[opt.name] = opt.deleteVideoFromCloudExtensions
s.cloudDiskUseStartupExtensions[opt.name] = opt.cloudDiskUseStartupExtensions
if(opt.onInsertTimelapseFrame)s.onInsertTimelapseFrame(opt.onInsertTimelapseFrame)
s.beforeAccountSave(opt.beforeAccountSave)
s.onAccountSave(opt.onAccountSave)
s.cloudDisksLoader(opt.name)
@ -13,6 +14,7 @@ module.exports = function(s){
s.loadGroupAppExtender(opt.loadGroupAppExtender)
s.unloadGroupAppExtender(opt.unloadGroupAppExtender)
s.insertCompletedVideoExtender(opt.insertCompletedVideoExtender)
if(opt.onInsertTimelapseFrame)s.onInsertTimelapseFrame(opt.onInsertTimelapseFrame)
s.beforeAccountSave(opt.beforeAccountSave)
s.onAccountSave(opt.onAccountSave)
s.onMonitorSave(opt.onMonitorSave)

67
libs/user.js
View File

@ -107,9 +107,9 @@ module.exports = function(s,config){
s.setDiskUsedForGroupAddStorage(e,{
size: -(frame.size/1000000),
storageIndex: storageIndex
})
},'timelapeFrames')
}else{
s.setDiskUsedForGroup(e,-(frame.size/1000000))
s.setDiskUsedForGroup(e,-(frame.size/1000000),'timelapeFrames')
}
// s.tx({
// f: 'timelapse_frame_delete',
@ -130,10 +130,10 @@ module.exports = function(s,config){
}
var deleteMainVideos = function(callback){
reRunCheck = function(){
callback()
return deleteMainVideos(callback)
}
//run purge command
if(s.group[e.ke].usedSpace > (s.group[e.ke].sizeLimit * config.cron.deleteOverMaxOffset)){
if(s.group[e.ke].usedSpaceVideos > (s.group[e.ke].sizeLimit * (s.group[e.ke].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)){
s.sqlQuery('SELECT * FROM Videos WHERE status != 0 AND details NOT LIKE \'%"archived":"1"%\' AND ke=? AND details NOT LIKE \'%"dir"%\' ORDER BY `time` ASC LIMIT 3',[e.ke],function(err,rows){
deleteSetOfVideos(err,rows,null,callback)
})
@ -175,10 +175,10 @@ module.exports = function(s,config){
}
var deleteTimelapseFrames = function(callback){
reRunCheck = function(){
callback()
return deleteTimelapseFrames(callback)
}
//run purge command
if(s.group[e.ke].usedSpace > (s.group[e.ke].sizeLimit * config.cron.deleteOverMaxOffset)){
if(s.group[e.ke].usedSpaceTimelapseFrames > (s.group[e.ke].sizeLimit * (s.group[e.ke].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){
s.sqlQuery('SELECT * FROM `Timelapse Frames` WHERE ke=? AND details NOT LIKE \'%"archived":"1"%\' ORDER BY `time` ASC LIMIT 3',[e.ke],function(err,frames){
deleteSetOfTimelapseFrames(err,frames,null,callback)
})
@ -186,20 +186,13 @@ module.exports = function(s,config){
callback()
}
}
var doAllChecks = function(){
deleteMainVideos(function(){
deleteTimelapseFrames(function(){
if(s.group[e.ke].usedSpace > (s.group[e.ke].sizeLimit * config.cron.deleteOverMaxOffset)){
doAllChecks()
}else{
deleteAddStorageVideos(function(){
finish()
})
}
deleteMainVideos(function(){
deleteTimelapseFrames(function(){
deleteAddStorageVideos(function(){
finish()
})
})
}
doAllChecks()
})
}
checkQueue()
}
@ -207,15 +200,15 @@ module.exports = function(s,config){
s.sendDiskUsedAmountToClients(e)
}
}
s.setDiskUsedForGroup = function(e,bytes){
s.setDiskUsedForGroup = function(e,bytes,storageType){
//`bytes` will be used as the value to add or substract
if(s.group[e.ke] && s.group[e.ke].diskUsedEmitter){
s.group[e.ke].diskUsedEmitter.emit('set',bytes)
s.group[e.ke].diskUsedEmitter.emit('set',bytes,storageType)
}
}
s.setDiskUsedForGroupAddStorage = function(e,data){
s.setDiskUsedForGroupAddStorage = function(e,data,storageType){
if(s.group[e.ke] && s.group[e.ke].diskUsedEmitter){
s.group[e.ke].diskUsedEmitter.emit('setAddStorage',data)
s.group[e.ke].diskUsedEmitter.emit('setAddStorage',data,storageType)
}
}
s.purgeCloudDiskForGroup = function(e,storageType){
@ -235,6 +228,10 @@ module.exports = function(s,config){
s.tx({
f: 'diskUsed',
size: s.group[e.ke].usedSpace,
usedSpace: s.group[e.ke].usedSpace,
usedSpaceVideos: s.group[e.ke].usedSpaceVideos,
usedSpaceFilebin: s.group[e.ke].usedSpaceFilebin,
usedSpaceTimelapseFrames: s.group[e.ke].usedSpaceTimelapseFrames,
limit: s.group[e.ke].sizeLimit,
addStorage: s.group[e.ke].addStorageUse
},'GRP_'+e.ke);
@ -268,6 +265,8 @@ module.exports = function(s,config){
if(!e.limit||e.limit===''){e.limit=10000}else{e.limit=parseFloat(e.limit)}
//save global space limit for group key (mb)
s.group[e.ke].sizeLimit = e.limit || s.group[e.ke].sizeLimit || 10000
s.group[e.ke].sizeLimitVideoPercent = parseFloat(s.group[e.ke].init.size_video_percent) || 90
s.group[e.ke].sizeLimitTimelapseFramesPercent = parseFloat(s.group[e.ke].init.size_timelapse_percent) || 10
//save global used space as megabyte value
s.group[e.ke].usedSpace = s.group[e.ke].usedSpace || ((e.size || 0) / 1000000)
//emit the changes to connected users
@ -349,7 +348,7 @@ module.exports = function(s,config){
}
})
//s.setDiskUsedForGroup
s.group[e.ke].diskUsedEmitter.on('set',function(currentChange){
s.group[e.ke].diskUsedEmitter.on('set',function(currentChange,storageType){
//validate current values
if(!s.group[e.ke].usedSpace){
s.group[e.ke].usedSpace=0
@ -361,6 +360,17 @@ module.exports = function(s,config){
}
//change global size value
s.group[e.ke].usedSpace += currentChange
switch(storageType){
case'timelapeFrames':
s.group[e.ke].usedSpaceTimelapseFrames += currentChange
break;
case'fileBin':
s.group[e.ke].usedSpaceFilebin += currentChange
break;
default:
s.group[e.ke].usedSpaceVideos += currentChange
break;
}
//remove value just used from queue
s.sendDiskUsedAmountToClients(e)
})
@ -378,6 +388,17 @@ module.exports = function(s,config){
}
//change global size value
storageIndex.usedSpace += currentSize
switch(storageType){
case'timelapeFrames':
storageIndex.usedSpaceTimelapseFrames += currentChange
break;
case'fileBin':
storageIndex.usedSpaceFilebin += currentChange
break;
default:
storageIndex.usedSpaceVideos += currentChange
break;
}
//remove value just used from queue
s.sendDiskUsedAmountToClients(e)
})

5
libs/videos.js
View File

@ -65,11 +65,6 @@ module.exports = function(s,config,lang){
v.details = details
})
}
//extender for "s.insertCompletedVideo"
s.insertCompletedVideoExtensions = []
s.insertCompletedVideoExtender = function(callback){
s.insertCompletedVideoExtensions.push(callback)
}
s.insertDatabaseRow = function(e,k,callback){
s.checkDetails(e)
//save database row

11
web/libs/js/dash2.socketio.js
View File

@ -744,9 +744,14 @@ $.ccio.globalWebsocket=function(d,user){
type: "time",
display: true,
time: {
format: timeFormat,
// round: 'day'
}
// format: timeFormat,
unit: 'minute',
displayFormats: {
minute: 'h:mm a',
},
},
categoryPercentage: 0.6,
barPercentage: .5,
}],
},
}

13
web/libs/js/dash2.timelapse.jpeg.js
View File

@ -14,16 +14,19 @@ $(document).ready(function(e){
$.timelapseJpeg.monitors=$.timelapseJpeg.e.find('.monitors_list')
$.timelapseJpeg.pointer = $.ccio.init('location',$user)
$.timelapseJpeg.downloadRecheckTimers = {}
$.timelapseJpeg.selectedStartDate = moment().utc().subtract(2, 'days').format('YYYY-MM-DD')
$.timelapseJpeg.selectedEndDate = moment().utc().format('YYYY-MM-DD')
$.timelapseJpeg.selectedStartDate = moment().utc().subtract(2, 'days').format('YYYY-MM-DDTHH:mm:ss')
$.timelapseJpeg.selectedEndDate = moment().utc().format('YYYY-MM-DDTHH:mm:ss')
$.timelapseJpeg.datepicker.daterangepicker({
showDropdowns: true,
startDate: moment().utc().subtract(2, 'days'),
endDate: moment().utc(),
timePicker: true,
locale: {
format: 'YYYY/MM/DD hh:mm:ss A'
}
}, function(start, end, label) {
console.log(start,end)
var selectedStartDate = moment(start).utc().format('YYYY-MM-DD')
var selectedEndDate = moment(end).utc().format('YYYY-MM-DD')
var selectedStartDate = moment(start).utc().format('YYYY-MM-DDTHH:mm:ss')
var selectedEndDate = moment(end).utc().format('YYYY-MM-DDTHH:mm:ss')
$.timelapseJpeg.draw(selectedStartDate,selectedEndDate)
$.timelapseJpeg.selectedStartDate = selectedStartDate
$.timelapseJpeg.selectedEndDate = selectedEndDate