diff --git a/libs/uploaders/loader.js b/libs/uploaders/loader.js index 2494ee64..985920c3 100644 --- a/libs/uploaders/loader.js +++ b/libs/uploaders/loader.js @@ -6,6 +6,7 @@ module.exports = function(s){ s.deleteVideoFromCloudExtensions[opt.name] = opt.deleteVideoFromCloudExtensions s.cloudDiskUseStartupExtensions[opt.name] = opt.cloudDiskUseStartupExtensions if(opt.onInsertTimelapseFrame)s.onInsertTimelapseFrame(opt.onInsertTimelapseFrame) + if(opt.onDeleteTimelapseFrameFromCloud)s.onDeleteTimelapseFrameFromCloudExtensions[opt.name] = opt.onDeleteTimelapseFrameFromCloud s.beforeAccountSave(opt.beforeAccountSave) s.onAccountSave(opt.onAccountSave) s.cloudDisksLoader(opt.name) diff --git a/libs/uploaders/s3based.js b/libs/uploaders/s3based.js index fe920bd6..f3db699e 100644 --- a/libs/uploaders/s3based.js +++ b/libs/uploaders/s3based.js @@ -34,8 +34,6 @@ module.exports = function(s,config,lang){ userDetails.whcs_accessKeyId !== ''&& userDetails.whcs_secretAccessKey && userDetails.whcs_secretAccessKey !== ''&& - userDetails.whcs_region && - userDetails.whcs_region !== ''&& userDetails.whcs_bucket !== '' ){ if(!userDetails.whcs_dir || userDetails.whcs_dir === '/'){ @@ -50,6 +48,10 @@ module.exports = function(s,config,lang){ if(!userDetails.whcs_endpoint || userDetails.whcs_endpoint === ''){ userDetails.whcs_endpoint = 's3.wasabisys.com' } + var whcs_region = null + if(userDetails.whcs_region && userDetails.whcs_region !== ''){ + whcs_region = userDetails.whcs_region + } var endpointSplit = userDetails.whcs_endpoint.split('.') if(endpointSplit.length > 2){ endpointSplit.shift() @@ -62,7 +64,7 @@ module.exports = function(s,config,lang){ endpoint: wasabiEndpoint, accessKeyId: userDetails.whcs_accessKeyId, secretAccessKey: userDetails.whcs_secretAccessKey, - region: userDetails.whcs_region + region: whcs_region }) s.group[e.ke].whcs = new s.group[e.ke].whcs.S3(); } @@ -112,6 +114,7 @@ module.exports = function(s,config,lang){ s.userLog(e,{type:lang['Wasabi Hot Cloud Storage Upload Error'],msg:err}) } if(s.group[e.ke].init.whcs_log === '1' && data && data.Location){ + console.log(data.Location) var save = [ e.mid, e.ke, @@ -135,47 +138,66 @@ module.exports = function(s,config,lang){ }) } } - var onInsertTimelapseFrame = function(monitorObject,frameDetails){ + var onInsertTimelapseFrame = function(monitorObject,queryInfo,filePath){ var e = monitorObject - // if(s.group[e.ke].whcs && s.group[e.ke].init.use_whcs !== '0' && s.group[e.ke].init.whcs_save === '1'){ - // var fileStream = fs.createReadStream(k.dir+k.filename); - // fileStream.on('error', function (err) { - // console.error(err) - // }) - // var saveLocation = s.group[e.ke].init.whcs_dir+e.ke+'/'+e.mid+'/'+k.filename - // s.group[e.ke].whcs.upload({ - // Bucket: s.group[e.ke].init.whcs_bucket, - // Key: saveLocation, - // Body:fileStream, - // ACL:'public-read', - // ContentType:'image/jpeg' - // },function(err,data){ - // if(err){ - // s.userLog(e,{type:lang['Wasabi Hot Cloud Storage Upload Error'],msg:err}) - // } - // if(s.group[e.ke].init.whcs_log === '1' && data && data.Location){ - // var save = [ - // e.mid, - // e.ke, - // k.startTime, - // 1, - // s.s({ - // type : 'whcs', - // location : saveLocation - // }), - // k.filesize, - // k.endTime, - // data.Location - // ] - // s.sqlQuery('INSERT INTO `Cloud Videos` (mid,ke,time,status,details,size,end,href) VALUES (?,?,?,?,?,?,?,?)',save) - // s.setCloudDiskUsedForGroup(e,{ - // amount : k.filesizeMB, - // storageType : 'whcs' - // }) - // s.purgeCloudDiskForGroup(e,'whcs') - // } - // }) - // } + if(s.group[e.ke].whcs && s.group[e.ke].init.use_whcs !== '0' && s.group[e.ke].init.whcs_save === '1'){ + var fileStream = fs.createReadStream(filePath) + fileStream.on('error', function (err) { + console.error(err) + }) + var saveLocation = s.group[e.ke].init.whcs_dir + e.ke + '/' + e.mid + '_timelapse/' + queryInfo.filename + s.group[e.ke].whcs.upload({ + Bucket: s.group[e.ke].init.whcs_bucket, + Key: saveLocation, + Body: fileStream, + ACL:'public-read', + ContentType:'image/jpeg' + },function(err,data){ + if(err){ + s.userLog(e,{type:lang['Wasabi Hot Cloud Storage Upload Error'],msg:err}) + } + if(s.group[e.ke].init.whcs_log === '1' && data && data.Location){ + var save = [ + queryInfo.mid, + queryInfo.ke, + queryInfo.time, + s.s({ + type : 'whcs', + location : saveLocation, + }), + queryInfo.size, + data.Location + ] + s.sqlQuery('INSERT INTO `Cloud Timelapse Frames` (mid,ke,time,details,size,href) VALUES (?,?,?,?,?,?,?,?)',save) + s.setCloudDiskUsedForGroup(e,{ + amount : s.kilobyteToMegabyte(queryInfo.size), + storageType : 'whcs' + },'timelapseFrames') + s.purgeCloudDiskForGroup(e,'whcs','timelapseFrames') + } + }) + } + } + var onDeleteTimelapseFrameFromCloud = function(e,frame,callback){ + // e = user + try{ + var frameDetails = JSON.parse(frame.details) + }catch(err){ + var frameDetails = frame.details + } + if(frameDetails.type !== 'whcs'){ + return + } + if(!frameDetails.location){ + frameDetails.location = frame.href.split(locationUrl)[1] + } + s.group[e.ke].whcs.deleteObject({ + Bucket: s.group[e.ke].init.whcs_bucket, + Key: frameDetails.location, + }, function(err, data) { + if (err) console.log(err); + callback() + }); } //wasabi s.addCloudUploader({ @@ -187,7 +209,8 @@ module.exports = function(s,config,lang){ cloudDiskUseStartupExtensions: cloudDiskUseStartupForWasabiHotCloudStorage, beforeAccountSave: beforeAccountSaveForWasabiHotCloudStorage, onAccountSave: cloudDiskUseStartupForWasabiHotCloudStorage, - onInsertTimelapseFrame: onInsertTimelapseFrame + onInsertTimelapseFrame: onInsertTimelapseFrame, + onDeleteTimelapseFrameFromCloud: onDeleteTimelapseFrameFromCloud }) return { "evaluation": "details.use_whcs !== '0'", @@ -288,6 +311,10 @@ module.exports = function(s,config,lang){ "default": "", "example": "", "possible": [ + { + "name": lang['No Region'], + "value": "" + }, { "name": "US West 1", "value": "us-west-1" diff --git a/libs/user.js b/libs/user.js index 20ece5f6..bdd8382e 100644 --- a/libs/user.js +++ b/libs/user.js @@ -202,26 +202,26 @@ module.exports = function(s,config){ s.sendDiskUsedAmountToClients(e) } } - s.setDiskUsedForGroup = function(e,bytes,storageType){ + s.setDiskUsedForGroup = function(e,bytes,storagePoint){ //`bytes` will be used as the value to add or substract if(s.group[e.ke] && s.group[e.ke].diskUsedEmitter){ - s.group[e.ke].diskUsedEmitter.emit('set',bytes,storageType) + s.group[e.ke].diskUsedEmitter.emit('set',bytes,storagePoint) } } - s.setDiskUsedForGroupAddStorage = function(e,data,storageType){ + s.setDiskUsedForGroupAddStorage = function(e,data,storagePoint){ if(s.group[e.ke] && s.group[e.ke].diskUsedEmitter){ - s.group[e.ke].diskUsedEmitter.emit('setAddStorage',data,storageType) + s.group[e.ke].diskUsedEmitter.emit('setAddStorage',data,storagePoint) } } - s.purgeCloudDiskForGroup = function(e,storageType){ + s.purgeCloudDiskForGroup = function(e,storageType,storagePoint){ if(s.group[e.ke].diskUsedEmitter){ - s.group[e.ke].diskUsedEmitter.emit('purgeCloud',storageType) + s.group[e.ke].diskUsedEmitter.emit('purgeCloud',storageType,storagePoint) } } - s.setCloudDiskUsedForGroup = function(e,usage){ - //`bytes` will be used as the value to add or substract + s.setCloudDiskUsedForGroup = function(e,usage,storagePoint){ + //`usage` will be used as the value to add or substract if(s.group[e.ke].diskUsedEmitter){ - s.group[e.ke].diskUsedEmitter.emit('setCloud',usage) + s.group[e.ke].diskUsedEmitter.emit('setCloud',usage,storagePoint) } } s.sendDiskUsedAmountToClients = function(e){ @@ -280,9 +280,9 @@ module.exports = function(s,config){ s.group[e.ke].init={}; } s.sqlQuery('SELECT * FROM Users WHERE ke=? AND details NOT LIKE ?',[e.ke,'%"sub"%'],function(ar,r){ - if(r&&r[0]){ - r=r[0]; - ar=JSON.parse(r.details); + if(r && r[0]){ + r = r[0]; + ar = JSON.parse(r.details); //load extenders s.loadGroupAppExtensions.forEach(function(extender){ extender(r,ar) @@ -290,7 +290,7 @@ module.exports = function(s,config){ //disk Used Emitter if(!s.group[e.ke].diskUsedEmitter){ s.group[e.ke].diskUsedEmitter = new events.EventEmitter() - s.group[e.ke].diskUsedEmitter.on('setCloud',function(currentChange){ + s.group[e.ke].diskUsedEmitter.on('setCloud',function(currentChange,storagePoint){ var amount = currentChange.amount var storageType = currentChange.storageType var cloudDisk = s.group[e.ke].cloudDiskUse[storageType] @@ -305,46 +305,90 @@ module.exports = function(s,config){ } //change global size value cloudDisk.usedSpace = cloudDisk.usedSpace + amount + switch(storagePoint){ + case'timelapeFrames': + cloudDisk.usedSpaceTimelapseFrames += amount + break; + case'fileBin': + cloudDisk.usedSpaceFilebin += amount + break; + default: + cloudDisk.usedSpaceVideos += amount + break; + } }) - s.group[e.ke].diskUsedEmitter.on('purgeCloud',function(storageType){ + s.group[e.ke].diskUsedEmitter.on('purgeCloud',function(storageType,storagePoint){ if(config.cron.deleteOverMax === true){ - //set queue processor - var finish=function(){ - // s.sendDiskUsedAmountToClients(e) - } - var deleteVideos = function(){ - //run purge command - var cloudDisk = s.group[e.ke].cloudDiskUse[storageType] - if(cloudDisk.sizeLimitCheck && cloudDisk.usedSpace > (cloudDisk.sizeLimit*config.cron.deleteOverMaxOffset)){ - s.sqlQuery('SELECT * FROM `Cloud Videos` WHERE status != 0 AND ke=? AND details LIKE \'%"type":"'+storageType+'"%\' ORDER BY `time` ASC LIMIT 2',[e.ke],function(err,videos){ - var videosToDelete = [] - var queryValues = [e.ke] - if(!videos)return console.log(err) - videos.forEach(function(video){ - video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext - videosToDelete.push('(mid=? AND `time`=?)') - queryValues.push(video.mid) - queryValues.push(video.time) - s.setCloudDiskUsedForGroup(e,{ - amount : -(video.size/1000000), - storageType : storageType - }) - s.deleteVideoFromCloudExtensionsRunner(e,storageType,video) + var cloudDisk = s.group[e.ke].cloudDiskUse[storageType] + //set queue processor + var finish=function(){ + // s.sendDiskUsedAmountToClients(e) + } + var deleteVideos = function(){ + //run purge command + if(cloudDisk.sizeLimitCheck && cloudDisk.usedSpace > (cloudDisk.sizeLimit*config.cron.deleteOverMaxOffset)){ + s.sqlQuery('SELECT * FROM `Cloud Videos` WHERE status != 0 AND ke=? AND details LIKE \'%"type":"'+storageType+'"%\' ORDER BY `time` ASC LIMIT 2',[e.ke],function(err,videos){ + var videosToDelete = [] + var queryValues = [e.ke] + if(!videos)return console.log(err) + videos.forEach(function(video){ + video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext + videosToDelete.push('(mid=? AND `time`=?)') + queryValues.push(video.mid) + queryValues.push(video.time) + s.setCloudDiskUsedForGroup(e,{ + amount : -(video.size/1000000), + storageType : storageType }) - if(videosToDelete.length > 0){ - videosToDelete = videosToDelete.join(' OR ') - s.sqlQuery('DELETE FROM `Cloud Videos` WHERE ke =? AND ('+videosToDelete+')',queryValues,function(){ - deleteVideos() - }) - }else{ - finish() - } + s.deleteVideoFromCloudExtensionsRunner(e,storageType,video) }) - }else{ - finish() - } + if(videosToDelete.length > 0){ + videosToDelete = videosToDelete.join(' OR ') + s.sqlQuery('DELETE FROM `Cloud Videos` WHERE ke =? AND ('+videosToDelete+')',queryValues,function(){ + deleteVideos() + }) + }else{ + finish() + } + }) + }else{ + finish() } - deleteVideos() + } + var deleteTimelapseFrames = function(callback){ + reRunCheck = function(){ + return deleteTimelapseFrames(callback) + } + //run purge command + if(cloudDisk.usedSpaceTimelapseFrames > (cloudDisk.sizeLimit * (s.group[e.ke].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){ + s.sqlQuery('SELECT * FROM `Cloud Timelapse Frames` WHERE ke=? AND details NOT LIKE \'%"archived":"1"%\' ORDER BY `time` ASC LIMIT 3',[e.ke],function(err,frames){ + var framesToDelete = [] + var queryValues = [e.ke] + if(!frames)return console.log(err) + frames.forEach(function(frame){ + frame.dir = s.getVideoDirectory(frame) + s.formattedTime(frame.time) + '.' + frame.ext + framesToDelete.push('(mid=? AND `time`=?)') + queryValues.push(frame.mid) + queryValues.push(frame.time) + s.setCloudDiskUsedForGroup(e,{ + amount : -(frame.size/1000000), + storageType : storageType + }) + s.deleteVideoFromCloudExtensionsRunner(e,storageType,frame) + }) + s.sqlQuery('DELETE FROM `Cloud Timelapse Frames` WHERE ke =? AND ('+framesToDelete+')',queryValues,function(){ + + }) + }) + }else{ + callback() + } + } + deleteVideos(function(){ + deleteTimelapseFrames(function(){ + + }) + }) }else{ // s.sendDiskUsedAmountToClients(e) } @@ -381,12 +425,12 @@ module.exports = function(s,config){ var storageIndex = data.storageIndex //validate current values if(!storageIndex.usedSpace){ - storageIndex.usedSpace=0 + storageIndex.usedSpace = 0 }else{ - storageIndex.usedSpace=parseFloat(storageIndex.usedSpace) + storageIndex.usedSpace = parseFloat(storageIndex.usedSpace) } - if(storageIndex.usedSpace<0||isNaN(storageIndex.usedSpace)){ - storageIndex.usedSpace=0 + if(storageIndex.usedSpace < 0 || isNaN(storageIndex.usedSpace)){ + storageIndex.usedSpace = 0 } //change global size value storageIndex.usedSpace += currentSize