ShinobiSystems
/
Shinobi
mirror of https://gitlab.com/Shinobi-Systems/Shinobi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

pass only groupKey when request Max Storage Amount check

const-conversion
Moe 2020-08-07 00:54:56 -07:00
parent 811b6fc846
commit 52a84f4501
6 changed files with 387 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libs/childNode.js
View File

@ -157,7 +157,7 @@ module.exports = function(s,config,lang,app,io){
extender(d.d,insert)
})
//purge over max
s.purgeDiskForGroup(d)
s.purgeDiskForGroup(d.ke)
//send new diskUsage values
s.setDiskUsedForGroup(d.ke,insert.filesizeMB)
clearTimeout(s.group[d.ke].activeMonitors[d.mid].recordingChecker)

2
libs/monitor.js
View File

@ -1043,7 +1043,7 @@ module.exports = function(s,config,lang){
switch(true){
case checkLog(d,'No space left on device'):
s.checkUserPurgeLock(e.ke)
s.purgeDiskForGroup(e)
s.purgeDiskForGroup(e.ke)
break;
case checkLog(d,'error while decoding'):
s.userLog(e,{type:lang['Error While Decoding'],msg:lang.ErrorWhileDecodingText});

2
libs/timelapse.js
View File

@ -80,7 +80,7 @@ module.exports = function(s,config,lang,app,io){
insert: queryInfo
})
s.setDiskUsedForGroup(e.ke,queryInfo.size / 1048576,'timelapeFrames')
s.purgeDiskForGroup(e)
s.purgeDiskForGroup(e.ke)
s.onInsertTimelapseFrameExtensions.forEach(function(extender){
extender(e,queryInfo,filePath)
})

377
libs/user.js
View File

@ -4,370 +4,21 @@ var spawn = require('child_process').spawn;
var exec = require('child_process').exec;
var async = require("async");
module.exports = function(s,config,lang){
const deleteSetOfVideos = function(options,callback){
const groupKey = options.groupKey
const err = options.err
const videos = options.videos
const storageIndex = options.storageIndex
const reRunCheck = options.reRunCheck
var completedCheck = 0
var whereGroup = []
var whereQuery = [
['ke','=',groupKey],
]
if(videos){
var didOne = false
videos.forEach(function(video){
video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext
if(didOne){
whereGroup.push(['or','mid','=',video.mid])
}else{
didOne = true
whereGroup.push(['mid','=',video.mid])
}
whereGroup.push(['time','=',video.time])
fs.chmod(video.dir,0o777,function(err){
fs.unlink(video.dir,function(err){
++completedCheck
if(err){
fs.stat(video.dir,function(err){
if(!err){
s.file('delete',video.dir)
}
})
}
const whereGroupLength = whereGroup.length / 2
if(whereGroupLength > 0 && whereGroupLength === completedCheck){
whereQuery[1] = whereGroup
s.knexQuery({
action: "delete",
table: "Videos",
where: whereQuery
},(err,info) => {
reRunCheck()
})
}
})
})
if(storageIndex){
s.setDiskUsedForGroupAddStorage(groupKey,{
size: -(video.size/1048576),
storageIndex: storageIndex
})
}else{
s.setDiskUsedForGroup(groupKey,-(video.size/1048576))
}
s.tx({
f: 'video_delete',
ff: 'over_max',
filename: s.formattedTime(video.time)+'.'+video.ext,
mid: video.mid,
ke: video.ke,
time: video.time,
end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')
},'GRP_'+groupKey)
})
}else{
console.log(err)
}
if(whereGroup.length === 0){
if(callback)callback()
}
}
var deleteSetOfTimelapseFrames = function(options,callback){
const groupKey = options.groupKey
const err = options.err
const frames = options.frames
const storageIndex = options.storageIndex
var whereGroup = []
var whereQuery = [
['ke','=',groupKey],
[]
]
var completedCheck = 0
if(frames){
var didOne = false
frames.forEach(function(frame){
var selectedDate = frame.filename.split('T')[0]
var dir = s.getTimelapseFrameDirectory(frame)
var fileLocationMid = `${dir}` + frame.filename
if(didOne){
whereGroup.push(
['or','mid','=',frame.mid],
['time','=',frame.time]
)
}else{
didOne = true
whereGroup.push(
['mid','=',frame.mid],
['time','=',frame.time]
)
}
fs.unlink(fileLocationMid,function(err){
++completedCheck
if(err){
fs.stat(fileLocationMid,function(err){
if(!err){
s.file('delete',fileLocationMid)
}
})
}
const whereGroupLength = whereGroup.length / 2
if(whereGroupLength > 0 && whereGroupLength === completedCheck){
whereQuery[1] = whereGroup
s.knexQuery({
action: "delete",
table: "Timelapse Frames",
where: whereQuery
},() => {
deleteTimelapseFrames(groupKey,callback)
})
}
})
if(storageIndex){
s.setDiskUsedForGroupAddStorage(groupKey,{
size: -(frame.size/1048576),
storageIndex: storageIndex
},'timelapeFrames')
}else{
s.setDiskUsedForGroup(groupKey,-(frame.size/1048576),'timelapeFrames')
}
// s.tx({
// f: 'timelapse_frame_delete',
// ff: 'over_max',
// filename: s.formattedTime(video.time)+'.'+video.ext,
// mid: video.mid,
// ke: video.ke,
// time: video.time,
// end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')
// },'GRP_'+groupKey)
})
}else{
console.log(err)
}
if(whereGroup.length === 0){
if(callback)callback()
}
}
var deleteSetOfFileBinFiles = function(options,callback){
const groupKey = options.groupKey
const err = options.err
const frames = options.frames
const storageIndex = options.storageIndex
var whereGroup = []
var whereQuery = [
['ke','=',groupKey],
[]
]
var completedCheck = 0
if(files){
files.forEach(function(file){
var dir = s.getFileBinDirectory(file)
var fileLocationMid = `${dir}` + file.name
if(whereGroup.length !== 0){
whereGroup.push(
['or','mid','=',file.mid],
['name','=',file.name]
)
}else{
whereGroup.push(
['mid','=',file.mid],
['name','=',file.name]
)
}
fs.unlink(fileLocationMid,function(err){
++completedCheck
if(err){
fs.stat(fileLocationMid,function(err){
if(!err){
s.file('delete',fileLocationMid)
}
})
}
const whereGroupLength = whereGroup.length / 2
if(whereGroupLength > 0 && whereGroupLength === completedCheck){
whereQuery[1] = whereGroup
s.knexQuery({
action: "delete",
table: "Files",
where: whereQuery
},() => {
deleteFileBinFiles(groupKey,callback)
})
}
})
if(storageIndex){
s.setDiskUsedForGroupAddStorage(groupKey,{
size: -(file.size/1048576),
storageIndex: storageIndex
},'fileBin')
}else{
s.setDiskUsedForGroup(groupKey,-(file.size/1048576),'fileBin')
}
})
}else{
console.log(err)
}
if(whereGroup.length === 0){
if(callback)callback()
}
}
var deleteAddStorageVideos = function(groupKey,callback){
reRunCheck = function(){
s.debugLog('deleteAddStorageVideos')
return deleteAddStorageVideos(groupKey,callback)
}
var currentStorageNumber = 0
var readStorageArray = function(){
setTimeout(function(){
reRunCheck = readStorageArray
var storage = s.listOfStorage[currentStorageNumber]
if(!storage){
//done all checks, move on to next user
callback()
return
}
var storageId = storage.value
if(storageId === '' || !s.group[groupKey].addStorageUse[storageId]){
++currentStorageNumber
readStorageArray()
return
}
var storageIndex = s.group[groupKey].addStorageUse[storageId]
//run purge command
if(storageIndex.usedSpace > (storageIndex.sizeLimit * (storageIndex.deleteOffset || config.cron.deleteOverMaxOffset))){
s.knexQuery({
action: "select",
columns: "*",
table: "Videos",
where: [
['ke','=',groupKey],
['status','!=','0'],
['details','NOT LIKE',`%"archived":"1"%`],
['details','LIKE',`%"dir":"${storage.value}"%`],
],
orderBy: ['time','asc'],
limit: 3
},(err,rows) => {
deleteSetOfVideos({
groupKey: groupKey,
err: err,
videos: rows,
storageIndex: storageIndex,
reRunCheck: () => {
return readStorageArray()
}
},callback)
})
}else{
++currentStorageNumber
readStorageArray()
}
})
}
readStorageArray()
}
var deleteMainVideos = function(groupKey,callback){
// //run purge command
// s.debugLog('!!!!!!!!!!!deleteMainVideos')
// s.debugLog('s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)')
// s.debugLog(s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset))
// s.debugLog('s.group[groupKey].usedSpaceVideos')
// s.debugLog(s.group[groupKey].usedSpaceVideos)
// s.debugLog('s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset')
// s.debugLog(s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)
// s.debugLog('s.group[groupKey].sizeLimitVideoPercent / 100')
// s.debugLog(s.group[groupKey].sizeLimitVideoPercent / 100)
// s.debugLog('s.group[groupKey].sizeLimit')
// s.debugLog(s.group[groupKey].sizeLimit)
if(s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Videos",
where: [
['ke','=',groupKey],
['status','!=','0'],
['details','NOT LIKE',`%"archived":"1"%`],
['details','NOT LIKE',`%"dir"%`],
],
orderBy: ['time','asc'],
limit: 3
},(err,rows) => {
deleteSetOfVideos({
groupKey: groupKey,
err: err,
videos: rows,
storageIndex: null,
reRunCheck: () => {
return deleteMainVideos(groupKey,callback)
}
},callback)
})
}else{
callback()
}
}
var deleteTimelapseFrames = function(groupKey,callback){
//run purge command
if(s.group[groupKey].usedSpaceTimelapseFrames > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Timelapse Frames",
where: [
['ke','=',groupKey],
['details','NOT LIKE',`%"archived":"1"%`],
],
orderBy: ['time','asc'],
limit: 3
},(err,frames) => {
deleteSetOfTimelapseFrames({
groupKey: groupKey,
err: err,
frames: frames,
storageIndex: null
},callback)
})
}else{
callback()
}
}
var deleteFileBinFiles = function(groupKey,callback){
if(config.deleteFileBinsOverMax === true){
//run purge command
if(s.group[groupKey].usedSpaceFileBin > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitFileBinPercent / 100) * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Files",
where: [
['ke','=',groupKey],
],
orderBy: ['time','asc'],
limit: 1
},(err,frames) => {
deleteSetOfFileBinFiles({
groupKey: groupKey,
err: err,
frames: frames,
storageIndex: null
},callback)
})
}else{
callback()
}
}else{
callback()
}
}
const {
deleteSetOfVideos,
deleteSetOfTimelapseFrames,
deleteSetOfFileBinFiles,
deleteAddStorageVideos,
deleteMainVideos,
deleteTimelapseFrames,
deleteFileBinFiles,
} = require("./user/utils.js")(s,config,lang);
let purgeDiskGroup = () => {}
const runQuery = async.queue(function(e, callback) {
purgeDiskGroup(e,callback)
const runQuery = async.queue(function(groupKey, callback) {
purgeDiskGroup(groupKey,callback)
}, 1);
if(config.cron.deleteOverMax === true){
purgeDiskGroup = (e,callback) => {
const groupKey = e.ke
purgeDiskGroup = (groupKey,callback) => {
if(s.group[groupKey]){
if(s.group[groupKey].sizePurging !== true){
s.group[groupKey].sizePurging = true
@ -392,8 +43,8 @@ module.exports = function(s,config,lang){
}
}
}
s.purgeDiskForGroup = (e) => {
return runQuery.push(e,function(){
s.purgeDiskForGroup = (groupKey) => {
return runQuery.push(groupKey,function(){
//...
})
}

369
libs/user/utils.js Normal file
View File

@ -0,0 +1,369 @@
var fs = require('fs');
module.exports = (s,config,lang) => {
const deleteSetOfVideos = function(options,callback){
const groupKey = options.groupKey
const err = options.err
const videos = options.videos
const storageIndex = options.storageIndex
const reRunCheck = options.reRunCheck
var completedCheck = 0
var whereGroup = []
var whereQuery = [
['ke','=',groupKey],
]
if(videos){
var didOne = false
videos.forEach(function(video){
video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext
if(didOne){
whereGroup.push(['or','mid','=',video.mid])
}else{
didOne = true
whereGroup.push(['mid','=',video.mid])
}
whereGroup.push(['time','=',video.time])
fs.chmod(video.dir,0o777,function(err){
fs.unlink(video.dir,function(err){
++completedCheck
if(err){
fs.stat(video.dir,function(err){
if(!err){
s.file('delete',video.dir)
}
})
}
const whereGroupLength = whereGroup.length / 2
if(whereGroupLength > 0 && whereGroupLength === completedCheck){
whereQuery[1] = whereGroup
s.knexQuery({
action: "delete",
table: "Videos",
where: whereQuery
},(err,info) => {
setTimeout(reRunCheck,1000)
})
}
})
})
if(storageIndex){
s.setDiskUsedForGroupAddStorage(groupKey,{
size: -(video.size/1048576),
storageIndex: storageIndex
})
}else{
s.setDiskUsedForGroup(groupKey,-(video.size/1048576))
}
s.tx({
f: 'video_delete',
ff: 'over_max',
filename: s.formattedTime(video.time)+'.'+video.ext,
mid: video.mid,
ke: video.ke,
time: video.time,
end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')
},'GRP_'+groupKey)
})
}else{
console.log(err)
}
if(whereGroup.length === 0){
if(callback)callback()
}
}
const deleteSetOfTimelapseFrames = function(options,callback){
const groupKey = options.groupKey
const err = options.err
const frames = options.frames
const storageIndex = options.storageIndex
var whereGroup = []
var whereQuery = [
['ke','=',groupKey],
[]
]
var completedCheck = 0
if(frames){
var didOne = false
frames.forEach(function(frame){
var selectedDate = frame.filename.split('T')[0]
var dir = s.getTimelapseFrameDirectory(frame)
var fileLocationMid = `${dir}` + frame.filename
if(didOne){
whereGroup.push(
['or','mid','=',frame.mid],
['time','=',frame.time]
)
}else{
didOne = true
whereGroup.push(
['mid','=',frame.mid],
['time','=',frame.time]
)
}
fs.unlink(fileLocationMid,function(err){
++completedCheck
if(err){
fs.stat(fileLocationMid,function(err){
if(!err){
s.file('delete',fileLocationMid)
}
})
}
const whereGroupLength = whereGroup.length / 2
if(whereGroupLength > 0 && whereGroupLength === completedCheck){
whereQuery[1] = whereGroup
s.knexQuery({
action: "delete",
table: "Timelapse Frames",
where: whereQuery
},() => {
deleteTimelapseFrames(groupKey,callback)
})
}
})
if(storageIndex){
s.setDiskUsedForGroupAddStorage(groupKey,{
size: -(frame.size/1048576),
storageIndex: storageIndex
},'timelapeFrames')
}else{
s.setDiskUsedForGroup(groupKey,-(frame.size/1048576),'timelapeFrames')
}
// s.tx({
// f: 'timelapse_frame_delete',
// ff: 'over_max',
// filename: s.formattedTime(video.time)+'.'+video.ext,
// mid: video.mid,
// ke: video.ke,
// time: video.time,
// end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')
// },'GRP_'+groupKey)
})
}else{
console.log(err)
}
if(whereGroup.length === 0){
if(callback)callback()
}
}
const deleteSetOfFileBinFiles = function(options,callback){
const groupKey = options.groupKey
const err = options.err
const frames = options.frames
const storageIndex = options.storageIndex
var whereGroup = []
var whereQuery = [
['ke','=',groupKey],
[]
]
var completedCheck = 0
if(files){
files.forEach(function(file){
var dir = s.getFileBinDirectory(file)
var fileLocationMid = `${dir}` + file.name
if(whereGroup.length !== 0){
whereGroup.push(
['or','mid','=',file.mid],
['name','=',file.name]
)
}else{
whereGroup.push(
['mid','=',file.mid],
['name','=',file.name]
)
}
fs.unlink(fileLocationMid,function(err){
++completedCheck
if(err){
fs.stat(fileLocationMid,function(err){
if(!err){
s.file('delete',fileLocationMid)
}
})
}
const whereGroupLength = whereGroup.length / 2
if(whereGroupLength > 0 && whereGroupLength === completedCheck){
whereQuery[1] = whereGroup
s.knexQuery({
action: "delete",
table: "Files",
where: whereQuery
},() => {
deleteFileBinFiles(groupKey,callback)
})
}
})
if(storageIndex){
s.setDiskUsedForGroupAddStorage(groupKey,{
size: -(file.size/1048576),
storageIndex: storageIndex
},'fileBin')
}else{
s.setDiskUsedForGroup(groupKey,-(file.size/1048576),'fileBin')
}
})
}else{
console.log(err)
}
if(whereGroup.length === 0){
if(callback)callback()
}
}
const deleteAddStorageVideos = function(groupKey,callback){
reRunCheck = function(){
s.debugLog('deleteAddStorageVideos')
return deleteAddStorageVideos(groupKey,callback)
}
var currentStorageNumber = 0
var readStorageArray = function(){
setTimeout(function(){
reRunCheck = readStorageArray
var storage = s.listOfStorage[currentStorageNumber]
if(!storage){
//done all checks, move on to next user
callback()
return
}
var storageId = storage.value
if(storageId === '' || !s.group[groupKey].addStorageUse[storageId]){
++currentStorageNumber
readStorageArray()
return
}
var storageIndex = s.group[groupKey].addStorageUse[storageId]
//run purge command
if(storageIndex.usedSpace > (storageIndex.sizeLimit * (storageIndex.deleteOffset || config.cron.deleteOverMaxOffset))){
s.knexQuery({
action: "select",
columns: "*",
table: "Videos",
where: [
['ke','=',groupKey],
['status','!=','0'],
['details','NOT LIKE',`%"archived":"1"%`],
['details','LIKE',`%"dir":"${storage.value}"%`],
],
orderBy: ['time','asc'],
limit: 3
},(err,rows) => {
deleteSetOfVideos({
groupKey: groupKey,
err: err,
videos: rows,
storageIndex: storageIndex,
reRunCheck: () => {
return readStorageArray()
}
},callback)
})
}else{
++currentStorageNumber
readStorageArray()
}
})
}
readStorageArray()
}
const deleteMainVideos = function(groupKey,callback){
// //run purge command
// s.debugLog('!!!!!!!!!!!deleteMainVideos')
// s.debugLog('s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)')
// s.debugLog(s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset))
// s.debugLog('s.group[groupKey].usedSpaceVideos')
// s.debugLog(s.group[groupKey].usedSpaceVideos)
// s.debugLog('s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset')
// s.debugLog(s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)
// s.debugLog('s.group[groupKey].sizeLimitVideoPercent / 100')
// s.debugLog(s.group[groupKey].sizeLimitVideoPercent / 100)
// s.debugLog('s.group[groupKey].sizeLimit')
// s.debugLog(s.group[groupKey].sizeLimit)
if(s.group[groupKey].usedSpaceVideos > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitVideoPercent / 100) * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Videos",
where: [
['ke','=',groupKey],
['status','!=','0'],
['details','NOT LIKE',`%"archived":"1"%`],
['details','NOT LIKE',`%"dir"%`],
],
orderBy: ['time','asc'],
limit: 3
},(err,rows) => {
deleteSetOfVideos({
groupKey: groupKey,
err: err,
videos: rows,
storageIndex: null,
reRunCheck: () => {
return deleteMainVideos(groupKey,callback)
}
},callback)
})
}else{
callback()
}
}
const deleteTimelapseFrames = function(groupKey,callback){
//run purge command
if(s.group[groupKey].usedSpaceTimelapseFrames > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Timelapse Frames",
where: [
['ke','=',groupKey],
['details','NOT LIKE',`%"archived":"1"%`],
],
orderBy: ['time','asc'],
limit: 3
},(err,frames) => {
deleteSetOfTimelapseFrames({
groupKey: groupKey,
err: err,
frames: frames,
storageIndex: null
},callback)
})
}else{
callback()
}
}
const deleteFileBinFiles = function(groupKey,callback){
if(config.deleteFileBinsOverMax === true){
//run purge command
if(s.group[groupKey].usedSpaceFileBin > (s.group[groupKey].sizeLimit * (s.group[groupKey].sizeLimitFileBinPercent / 100) * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Files",
where: [
['ke','=',groupKey],
],
orderBy: ['time','asc'],
limit: 1
},(err,frames) => {
deleteSetOfFileBinFiles({
groupKey: groupKey,
err: err,
frames: frames,
storageIndex: null
},callback)
})
}else{
callback()
}
}else{
callback()
}
}
return {
deleteSetOfVideos: deleteSetOfVideos,
deleteSetOfTimelapseFrames: deleteSetOfTimelapseFrames,
deleteSetOfFileBinFiles: deleteSetOfFileBinFiles,
deleteAddStorageVideos: deleteAddStorageVideos,
deleteMainVideos: deleteMainVideos,
deleteTimelapseFrames: deleteTimelapseFrames,
deleteFileBinFiles: deleteFileBinFiles,
}
}

2
libs/videos.js
View File

@ -172,7 +172,7 @@ module.exports = function(s,config,lang){
events: k.events && k.events.length > 0 ? k.events : null
},'GRP_'+e.ke,'video_view')
//purge over max
s.purgeDiskForGroup(e)
s.purgeDiskForGroup(e.ke)
//send new diskUsage values
var storageIndex = s.getVideoStorageIndex(e)
if(storageIndex){