update libs/uploaders + for timelapse frames

- update user.js
+ add "No Region" for s3 based uploader

libs/uploaders/loader.js

@@ -6,6 +6,7 @@ module.exports = function(s){
         s.deleteVideoFromCloudExtensions[opt.name] = opt.deleteVideoFromCloudExtensions
         s.cloudDiskUseStartupExtensions[opt.name] = opt.cloudDiskUseStartupExtensions
         if(opt.onInsertTimelapseFrame)s.onInsertTimelapseFrame(opt.onInsertTimelapseFrame)
+        if(opt.onDeleteTimelapseFrameFromCloud)s.onDeleteTimelapseFrameFromCloudExtensions[opt.name] = opt.onDeleteTimelapseFrameFromCloud
         s.beforeAccountSave(opt.beforeAccountSave)
         s.onAccountSave(opt.onAccountSave)
         s.cloudDisksLoader(opt.name)

libs/uploaders/s3based.js

@@ -34,8 +34,6 @@ module.exports = function(s,config,lang){
            userDetails.whcs_accessKeyId !== ''&&
            userDetails.whcs_secretAccessKey &&
            userDetails.whcs_secretAccessKey !== ''&&
-           userDetails.whcs_region &&
-           userDetails.whcs_region !== ''&&
            userDetails.whcs_bucket !== ''
           ){
             if(!userDetails.whcs_dir || userDetails.whcs_dir === '/'){
@@ -50,6 +48,10 @@ module.exports = function(s,config,lang){
             if(!userDetails.whcs_endpoint || userDetails.whcs_endpoint === ''){
                 userDetails.whcs_endpoint = 's3.wasabisys.com'
             }
+            var whcs_region = null
+            if(userDetails.whcs_region && userDetails.whcs_region !== ''){
+                whcs_region = userDetails.whcs_region
+            }
             var endpointSplit = userDetails.whcs_endpoint.split('.')
             if(endpointSplit.length > 2){
                 endpointSplit.shift()
@@ -62,7 +64,7 @@ module.exports = function(s,config,lang){
                 endpoint: wasabiEndpoint,
                 accessKeyId: userDetails.whcs_accessKeyId,
                 secretAccessKey: userDetails.whcs_secretAccessKey,
-                region: userDetails.whcs_region
+                region: whcs_region
             })
             s.group[e.ke].whcs = new s.group[e.ke].whcs.S3();
         }
@@ -112,6 +114,7 @@ module.exports = function(s,config,lang){
                     s.userLog(e,{type:lang['Wasabi Hot Cloud Storage Upload Error'],msg:err})
                 }
                 if(s.group[e.ke].init.whcs_log === '1' && data && data.Location){
+                    console.log(data.Location)
                     var save = [
                         e.mid,
                         e.ke,
@@ -135,47 +138,66 @@ module.exports = function(s,config,lang){
             })
         }
     }
-    var onInsertTimelapseFrame = function(monitorObject,frameDetails){
+    var onInsertTimelapseFrame = function(monitorObject,queryInfo,filePath){
         var e = monitorObject
-        // if(s.group[e.ke].whcs && s.group[e.ke].init.use_whcs !== '0' && s.group[e.ke].init.whcs_save === '1'){
+        if(s.group[e.ke].whcs && s.group[e.ke].init.use_whcs !== '0' && s.group[e.ke].init.whcs_save === '1'){
-        //     var fileStream = fs.createReadStream(k.dir+k.filename);
+            var fileStream = fs.createReadStream(filePath)
-        //     fileStream.on('error', function (err) {
+            fileStream.on('error', function (err) {
-        //         console.error(err)
+                console.error(err)
-        //     })
+            })
-        //     var saveLocation = s.group[e.ke].init.whcs_dir+e.ke+'/'+e.mid+'/'+k.filename
+            var saveLocation = s.group[e.ke].init.whcs_dir + e.ke + '/' + e.mid + '_timelapse/' + queryInfo.filename
-        //     s.group[e.ke].whcs.upload({
+            s.group[e.ke].whcs.upload({
-        //         Bucket: s.group[e.ke].init.whcs_bucket,
+                Bucket: s.group[e.ke].init.whcs_bucket,
-        //         Key: saveLocation,
+                Key: saveLocation,
-        //         Body:fileStream,
+                Body: fileStream,
-        //         ACL:'public-read',
+                ACL:'public-read',
-        //         ContentType:'image/jpeg'
+                ContentType:'image/jpeg'
-        //     },function(err,data){
+            },function(err,data){
-        //         if(err){
+                if(err){
-        //             s.userLog(e,{type:lang['Wasabi Hot Cloud Storage Upload Error'],msg:err})
+                    s.userLog(e,{type:lang['Wasabi Hot Cloud Storage Upload Error'],msg:err})
-        //         }
+                }
-        //         if(s.group[e.ke].init.whcs_log === '1' && data && data.Location){
+                if(s.group[e.ke].init.whcs_log === '1' && data && data.Location){
-        //             var save = [
+                    var save = [
-        //                 e.mid,
+                        queryInfo.mid,
-        //                 e.ke,
+                        queryInfo.ke,
-        //                 k.startTime,
+                        queryInfo.time,
-        //                 1,
+                        s.s({
-        //                 s.s({
+                            type : 'whcs',
-        //                     type : 'whcs',
+                            location : saveLocation,
-        //                     location : saveLocation
+                        }),
-        //                 }),
+                        queryInfo.size,
-        //                 k.filesize,
+                        data.Location
-        //                 k.endTime,
+                    ]
-        //                 data.Location
+                    s.sqlQuery('INSERT INTO `Cloud Timelapse Frames` (mid,ke,time,details,size,href) VALUES (?,?,?,?,?,?,?,?)',save)
-        //             ]
+                    s.setCloudDiskUsedForGroup(e,{
-        //             s.sqlQuery('INSERT INTO `Cloud Videos` (mid,ke,time,status,details,size,end,href) VALUES (?,?,?,?,?,?,?,?)',save)
+                        amount : s.kilobyteToMegabyte(queryInfo.size),
-        //             s.setCloudDiskUsedForGroup(e,{
+                        storageType : 'whcs'
-        //                 amount : k.filesizeMB,
+                    },'timelapseFrames')
-        //                 storageType : 'whcs'
+                    s.purgeCloudDiskForGroup(e,'whcs','timelapseFrames')
-        //             })
+                }
-        //             s.purgeCloudDiskForGroup(e,'whcs')
+            })
-        //         }
+        }
-        //     })
+    }
-        // }
+    var onDeleteTimelapseFrameFromCloud = function(e,frame,callback){
+        // e = user
+        try{
+            var frameDetails = JSON.parse(frame.details)
+        }catch(err){
+            var frameDetails = frame.details
+        }
+        if(frameDetails.type !== 'whcs'){
+            return
+        }
+        if(!frameDetails.location){
+            frameDetails.location = frame.href.split(locationUrl)[1]
+        }
+        s.group[e.ke].whcs.deleteObject({
+            Bucket: s.group[e.ke].init.whcs_bucket,
+            Key: frameDetails.location,
+        }, function(err, data) {
+            if (err) console.log(err);
+            callback()
+        });
     }
     //wasabi
     s.addCloudUploader({
@@ -187,7 +209,8 @@ module.exports = function(s,config,lang){
         cloudDiskUseStartupExtensions: cloudDiskUseStartupForWasabiHotCloudStorage,
         beforeAccountSave: beforeAccountSaveForWasabiHotCloudStorage,
         onAccountSave: cloudDiskUseStartupForWasabiHotCloudStorage,
-        onInsertTimelapseFrame: onInsertTimelapseFrame
+        onInsertTimelapseFrame: onInsertTimelapseFrame,
+        onDeleteTimelapseFrameFromCloud: onDeleteTimelapseFrameFromCloud
     })
     return {
        "evaluation": "details.use_whcs !== '0'",
@@ -288,6 +311,10 @@ module.exports = function(s,config,lang){
               "default": "",
               "example": "",
               "possible": [
+                   {
+                      "name": lang['No Region'],
+                      "value": ""
+                   },
                    {
                       "name": "US West 1",
                       "value": "us-west-1"

libs/user.js

@@ -202,26 +202,26 @@ module.exports = function(s,config){
             s.sendDiskUsedAmountToClients(e)
         }
     }
-    s.setDiskUsedForGroup = function(e,bytes,storageType){
+    s.setDiskUsedForGroup = function(e,bytes,storagePoint){
         //`bytes` will be used as the value to add or substract
         if(s.group[e.ke] && s.group[e.ke].diskUsedEmitter){
-            s.group[e.ke].diskUsedEmitter.emit('set',bytes,storageType)
+            s.group[e.ke].diskUsedEmitter.emit('set',bytes,storagePoint)
         }
     }
-    s.setDiskUsedForGroupAddStorage = function(e,data,storageType){
+    s.setDiskUsedForGroupAddStorage = function(e,data,storagePoint){
         if(s.group[e.ke] && s.group[e.ke].diskUsedEmitter){
-            s.group[e.ke].diskUsedEmitter.emit('setAddStorage',data,storageType)
+            s.group[e.ke].diskUsedEmitter.emit('setAddStorage',data,storagePoint)
         }
     }
-    s.purgeCloudDiskForGroup = function(e,storageType){
+    s.purgeCloudDiskForGroup = function(e,storageType,storagePoint){
         if(s.group[e.ke].diskUsedEmitter){
-            s.group[e.ke].diskUsedEmitter.emit('purgeCloud',storageType)
+            s.group[e.ke].diskUsedEmitter.emit('purgeCloud',storageType,storagePoint)
         }
     }
-    s.setCloudDiskUsedForGroup = function(e,usage){
+    s.setCloudDiskUsedForGroup = function(e,usage,storagePoint){
-        //`bytes` will be used as the value to add or substract
+        //`usage` will be used as the value to add or substract
         if(s.group[e.ke].diskUsedEmitter){
-            s.group[e.ke].diskUsedEmitter.emit('setCloud',usage)
+            s.group[e.ke].diskUsedEmitter.emit('setCloud',usage,storagePoint)
         }
     }
     s.sendDiskUsedAmountToClients = function(e){
@@ -290,7 +290,7 @@ module.exports = function(s,config){
                 //disk Used Emitter
                 if(!s.group[e.ke].diskUsedEmitter){
                     s.group[e.ke].diskUsedEmitter = new events.EventEmitter()
-                    s.group[e.ke].diskUsedEmitter.on('setCloud',function(currentChange){
+                    s.group[e.ke].diskUsedEmitter.on('setCloud',function(currentChange,storagePoint){
                         var amount = currentChange.amount
                         var storageType = currentChange.storageType
                         var cloudDisk = s.group[e.ke].cloudDiskUse[storageType]
@@ -305,16 +305,27 @@ module.exports = function(s,config){
                         }
                         //change global size value
                         cloudDisk.usedSpace = cloudDisk.usedSpace + amount
+                        switch(storagePoint){
+                            case'timelapeFrames':
+                                cloudDisk.usedSpaceTimelapseFrames += amount
+                            break;
+                            case'fileBin':
+                                cloudDisk.usedSpaceFilebin += amount
+                            break;
+                            default:
+                                cloudDisk.usedSpaceVideos += amount
+                            break;
+                        }
                     })
-                    s.group[e.ke].diskUsedEmitter.on('purgeCloud',function(storageType){
+                    s.group[e.ke].diskUsedEmitter.on('purgeCloud',function(storageType,storagePoint){
                         if(config.cron.deleteOverMax === true){
+                            var cloudDisk = s.group[e.ke].cloudDiskUse[storageType]
                             //set queue processor
                             var finish=function(){
                                 // s.sendDiskUsedAmountToClients(e)
                             }
                             var deleteVideos = function(){
                                 //run purge command
-                                    var cloudDisk = s.group[e.ke].cloudDiskUse[storageType]
                                 if(cloudDisk.sizeLimitCheck && cloudDisk.usedSpace > (cloudDisk.sizeLimit*config.cron.deleteOverMaxOffset)){
                                         s.sqlQuery('SELECT * FROM `Cloud Videos` WHERE status != 0 AND ke=? AND details LIKE \'%"type":"'+storageType+'"%\' ORDER BY `time` ASC LIMIT 2',[e.ke],function(err,videos){
                                             var videosToDelete = []
@@ -344,7 +355,40 @@ module.exports = function(s,config){
                                     finish()
                                 }
                             }
-                                deleteVideos()
+                            var deleteTimelapseFrames = function(callback){
+                                reRunCheck = function(){
+                                    return deleteTimelapseFrames(callback)
+                                }
+                                //run purge command
+                                if(cloudDisk.usedSpaceTimelapseFrames > (cloudDisk.sizeLimit * (s.group[e.ke].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){
+                                    s.sqlQuery('SELECT * FROM `Cloud Timelapse Frames` WHERE ke=? AND details NOT LIKE \'%"archived":"1"%\' ORDER BY `time` ASC LIMIT 3',[e.ke],function(err,frames){
+                                        var framesToDelete = []
+                                        var queryValues = [e.ke]
+                                        if(!frames)return console.log(err)
+                                        frames.forEach(function(frame){
+                                            frame.dir = s.getVideoDirectory(frame) + s.formattedTime(frame.time) + '.' + frame.ext
+                                            framesToDelete.push('(mid=? AND `time`=?)')
+                                            queryValues.push(frame.mid)
+                                            queryValues.push(frame.time)
+                                            s.setCloudDiskUsedForGroup(e,{
+                                                amount : -(frame.size/1000000),
+                                                storageType : storageType
+                                            })
+                                            s.deleteVideoFromCloudExtensionsRunner(e,storageType,frame)
+                                        })
+                                        s.sqlQuery('DELETE FROM `Cloud Timelapse Frames` WHERE ke =? AND ('+framesToDelete+')',queryValues,function(){
+
+                                        })
+                                    })
+                                }else{
+                                    callback()
+                                }
+                            }
+                            deleteVideos(function(){
+                                deleteTimelapseFrames(function(){
+
+                                })
+                            })
                         }else{
                             // s.sendDiskUsedAmountToClients(e)
                         }