From 91fadc44dffb5e586860245674155d1852588c4a Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Sun, 13 Sep 2020 23:49:52 -0700 Subject: [PATCH 1/5] P2P : Can create connection based on config --- .gitignore | 1 + camera.js | 6 +- libs/commander.js | 166 +++++++ libs/socketio.js | 33 +- libs/startup.js | 803 +++++++++++++++++----------------- web/libs/js/dash2.socketio.js | 7 +- web/pages/blocks/header.ejs | 2 +- 7 files changed, 595 insertions(+), 423 deletions(-) create mode 100644 libs/commander.js diff --git a/.gitignore b/.gitignore index c008a49d..cd1cef74 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ npm-debug.log shinobi.sqlite package-lock.json dist +._* diff --git a/camera.js b/camera.js index de6657fd..f200e22b 100644 --- a/camera.js +++ b/camera.js @@ -26,7 +26,7 @@ require('./libs/codeTester.js')(s,config,lang) //get version require('./libs/version.js')(s,config,lang) //video processing engine -require('./libs/ffmpeg.js')(s,config,lang,function(ffmpeg){ +require('./libs/ffmpeg.js')(s,config,lang,async function(ffmpeg){ //ffmpeg coProcessor require('./libs/ffmpegCoProcessor.js')(s,config,lang,ffmpeg) //database connection : mysql, sqlite3.. @@ -86,5 +86,7 @@ require('./libs/ffmpeg.js')(s,config,lang,function(ffmpeg){ //scheduling engine require('./libs/scheduler.js')(s,config,lang,app,io) //on-start actions, daemon(s) starter - require('./libs/startup.js')(s,config,lang) + await require('./libs/startup.js')(s,config,lang) + //p2p, commander + require('./libs/commander.js')(s,config,lang) }) diff --git a/libs/commander.js b/libs/commander.js new file mode 100644 index 00000000..b05d545d --- /dev/null +++ b/libs/commander.js @@ -0,0 +1,166 @@ +module.exports = function(s,config,lang,app,io){ + // Shinobi P2P Client Script + if(config.p2pEnabled){ + if(!config.p2pHost)config.p2pHost = 'ws://163.172.180.205:8084' + var p2pClientConnectionStaticName = 'Commander' + var p2pClientConnections = {} + var runningRequests = {} + var request = require('request'); + const socketIOClient = require('socket.io-client'); + var parseJSON = function(string){ + var parsed = string + try{ + parsed = JSON.parse(string) + }catch(err){ + + } + return parsed + } + var createQueryStringFromObject = function(obj){ + var queryString = '' + var keys = Object.keys(obj) + keys.forEach(function(key){ + var value = obj[key] + queryString += `&${key}=${value}` + }) + return queryString + } + var doRequest = function(url,method,data,callback,onDataReceived){ + var requestEndpoint = url + if(method === 'GET' && data){ + requestEndpoint += '?' + createQueryStringFromObject(data) + } + return request(requestEndpoint,{ + method: method, + json: method !== 'GET' ? (data ? data : null) : null + }, function(err,resp,body){ + // var json = parseJSON(body) + if(err)console.error(err,data) + callback(err,body,resp) + }).on('data', function(data) { + onDataReceived(data) + }) + } + var createShinobiSocketConnection = function(connectionId){ + var masterConnectionToMachine = socketIOClient(`ws://localhost:${config.port}`, {transports:['websocket']}) + p2pClientConnections[connectionId || p2pClientConnectionStaticName] = masterConnectionToMachine + return masterConnectionToMachine + } + // + const createSocketConnections = () => { + console.log(`Connecting to ${config.p2pHost}`) + const connectionToP2PServer = socketIOClient(config.p2pHost, {transports:['websocket']}); + if(!config.p2pApiKey){ + console.log(`Please fill 'p2pApiKey' in your conf.json.`) + } + if(!config.p2pGroupId){ + console.log(`Please fill 'p2pGroupId' in your conf.json.`) + } + config.machineId = config.machineId || s.gid(20) + connectionToP2PServer.on('connect',function(){ + console.log(`Connected ${config.p2pHost}`) + connectionToP2PServer.emit('initMachine',{ + port: config.port, + apiKey: config.p2pApiKey, + groupId: config.p2pGroupId, + targetUserId: config.p2pTargetUserId, + targetGroupId: config.p2pTargetGroupId, + machineId: config.machineId, + subscriptionId: config.subscriptionId || 'notActivated' + }) + }) + connectionToP2PServer.on('httpClose',function(requestId){ + if(runningRequests[requestId] && runningRequests[requestId].abort){ + runningRequests[requestId].abort() + } + }) + connectionToP2PServer.on('http',function(rawRequest){ + runningRequests[rawRequest.rid] = doRequest( + rawRequest.url, + rawRequest.method, + rawRequest.data, + function(err,json,resp){ + connectionToP2PServer.emit('httpResponse',{ + err: err, + json: rawRequest.bodyOnEnd ? json : null, + rid: rawRequest.rid + }) + }, + function(data){ + if(!rawRequest.bodyOnEnd)connectionToP2PServer.emit('httpResponseChunk',{ + data: data, + rid: rawRequest.rid + }) + }) + }) + const masterConnectionToMachine = createShinobiSocketConnection() + masterConnectionToMachine.on('connect',function(){ + masterConnectionToMachine.emit('f',{ + f: 'init', + ke: config.p2pTargetGroupId, + uid: config.p2pTargetUserId + }) + }) + masterConnectionToMachine.on('f',function(data){ + connectionToP2PServer.emit('f',data) + }) + + connectionToP2PServer.on('wsInit',function(rawRequest){ + console.log('wsInit') + var user = rawRequest.user + var clientConnectionToMachine = createShinobiSocketConnection(rawRequest.cnid) + connectionToP2PServer.on('f',function(rawRequest){ + clientConnectionToMachine.emit('f',rawRequest.data) + }) + clientConnectionToMachine.on('connect',function(){ + clientConnectionToMachine.emit('f',{ + f: 'init', + ke: user.ke, + uid: user.uid, + }) + }) + clientConnectionToMachine.on('f',function(data){ + connectionToP2PServer.emit('f',{data: data, cnid: rawRequest.cnid}) + }) + }); + connectionToP2PServer.on('wsDestroy',function(rawRequest){ + if(p2pClientConnections[rawRequest.cnid]){ + p2pClientConnections[rawRequest.cnid].disconnect(); + } + delete(p2pClientConnections[rawRequest.cnid]) + }); + + connectionToP2PServer.on('allowDisconnect',function(bool){ + connectionToP2PServer.allowDisconnect = true; + connectionToP2PServer.disconnect() + console.log('Server said to go away') + }); + const onDisconnect = () => { + console.log('disconnected p2p') + if(!connectionToP2PServer.allowDisconnect){ + setTimeout(function(){ + connectionToP2PServer.connect() + },3000) + } + } + connectionToP2PServer.on('error',onDisconnect) + connectionToP2PServer.on('disconnect',onDisconnect) + } + if(config.p2pTargetGroupId && config.p2pTargetUserId){ + createSocketConnections() + }else{ + s.knexQuery({ + action: "select", + columns: "ke,uid", + table: "Users", + where: [], + limit: 1 + },(err,r) => { + const firstUser = r[0] + config.p2pTargetUserId = firstUser.uid + config.p2pTargetGroupId = firstUser.ke + createSocketConnections() + }) + } + } +} diff --git a/libs/socketio.js b/libs/socketio.js index 34bddd9d..731e4c7a 100644 --- a/libs/socketio.js +++ b/libs/socketio.js @@ -51,17 +51,21 @@ module.exports = function(s,config,lang,io){ } } - const streamConnectionAuthentication = (options) => { + const streamConnectionAuthentication = (options,ipAddress) => { return new Promise( (resolve,reject) => { + var isInternal = false + if(ipAddress.indexOf('localhost') > -1 || ipAddress.indexOf('127.0.0.1') > -1){ + isInternal = true + } + const baseWheres = [ + ['ke','=',options.ke], + ['uid','=',options.uid], + ] s.knexQuery({ action: "select", columns: "ke,uid,auth,mail,details", table: "Users", - where: [ - ['ke','=',options.ke], - ['auth','=',options.auth], - ['uid','=',options.uid], - ] + where: baseWheres.concat(!isInternal ? [['auth','=',options.auth]] : []) },(err,r) => { if(r&&r[0]){ resolve(r) @@ -70,11 +74,7 @@ module.exports = function(s,config,lang,io){ action: "select", columns: "*", table: "API", - where: [ - ['ke','=',options.ke], - ['code','=',options.auth], - ['uid','=',options.uid], - ] + where: baseWheres.concat(!isInternal ? [['code','=',options.auth]] : []) },(err,r) => { if(r && r[0]){ r = r[0] @@ -165,7 +165,7 @@ module.exports = function(s,config,lang,io){ if(s.group[d.ke]&&s.group[d.ke].users&&s.group[d.ke].users[d.auth]){ onSuccess(s.group[d.ke].users[d.auth]); }else{ - streamConnectionAuthentication(d).then(onSuccess).catch(onFail) + streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail) } }) //unique Base64 socket stream @@ -198,7 +198,7 @@ module.exports = function(s,config,lang,io){ if(s.group[d.ke]&&s.group[d.ke].users&&s.group[d.ke].users[d.auth]){ onSuccess(s.group[d.ke].users[d.auth]); }else{ - streamConnectionAuthentication(d).then(onSuccess).catch(onFail) + streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail) } }) //unique FLV socket stream @@ -231,7 +231,7 @@ module.exports = function(s,config,lang,io){ if(s.group[d.ke] && s.group[d.ke].users && s.group[d.ke].users[d.auth]){ onSuccess(s.group[d.ke].users[d.auth]); }else{ - streamConnectionAuthentication(d).then(onSuccess).catch(onFail) + streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail) } }) //unique MP4 socket stream @@ -312,7 +312,7 @@ module.exports = function(s,config,lang,io){ if(s.group[d.ke]&&s.group[d.ke].users&&s.group[d.ke].users[d.auth]){ onSuccess(s.group[d.ke].users[d.auth]); }else{ - streamConnectionAuthentication(d).then(onSuccess).catch(onFail) + streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail) } }) //main socket control functions @@ -393,7 +393,8 @@ module.exports = function(s,config,lang,io){ extender(r,cn,d,tx) }) } - streamConnectionAuthentication(d).then(onSuccess).catch(onFail) + console.log(cn.ip) + streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail) return; } if((d.id||d.uid||d.mid)&&cn.ke){ diff --git a/libs/startup.js b/libs/startup.js index cf4bcf5d..8cb64499 100644 --- a/libs/startup.js +++ b/libs/startup.js @@ -6,443 +6,446 @@ var crypto = require('crypto'); var exec = require('child_process').exec; var execSync = require('child_process').execSync; module.exports = function(s,config,lang,io){ - var checkedAdminUsers = {} - console.log('FFmpeg version : '+s.ffmpegVersion) - console.log('Node.js version : '+process.version) - s.processReady = function(){ - delete(checkedAdminUsers) - s.systemLog(lang.startUpText5) - s.onProcessReadyExtensions.forEach(function(extender){ - extender(true) - }) - process.send('ready') - } - var checkForTerminalCommands = function(callback){ - var next = function(){ - if(callback)callback() - } - if(!s.isWin && s.packageJson.mainDirectory !== '.'){ - var etcPath = '/etc/shinobisystems/cctv.txt' - fs.stat(etcPath,function(err,stat){ - if(err || !stat){ - exec('node '+ s.mainDirectory + '/INSTALL/terminalCommands.js',function(err){ - if(err)console.log(err) - }) - } - next() + return new Promise((resolve, reject) => { + var checkedAdminUsers = {} + console.log('FFmpeg version : '+s.ffmpegVersion) + console.log('Node.js version : '+process.version) + s.processReady = function(){ + delete(checkedAdminUsers) + resolve() + s.systemLog(lang.startUpText5) + s.onProcessReadyExtensions.forEach(function(extender){ + extender(true) }) - }else{ - next() + process.send('ready') } - } - var loadedAccounts = [] - var foundMonitors = [] - var loadMonitors = function(callback){ - s.beforeMonitorsLoadedOnStartupExtensions.forEach(function(extender){ - extender() - }) - s.systemLog(lang.startUpText4) - //preliminary monitor start - s.knexQuery({ - action: "select", - columns: "*", - table: "Monitors", - },function(err,monitors) { - foundMonitors = monitors - if(err){s.systemLog(err)} - if(monitors && monitors[0]){ - var didNotLoad = 0 - var loadCompleted = 0 - var orphanedVideosForMonitors = {} - var loadMonitor = function(monitor){ - const checkAnother = function(){ - ++loadCompleted - if(monitors[loadCompleted]){ - loadMonitor(monitors[loadCompleted]) + var checkForTerminalCommands = function(callback){ + var next = function(){ + if(callback)callback() + } + if(!s.isWin && s.packageJson.mainDirectory !== '.'){ + var etcPath = '/etc/shinobisystems/cctv.txt' + fs.stat(etcPath,function(err,stat){ + if(err || !stat){ + exec('node '+ s.mainDirectory + '/INSTALL/terminalCommands.js',function(err){ + if(err)console.log(err) + }) + } + next() + }) + }else{ + next() + } + } + var loadedAccounts = [] + var foundMonitors = [] + var loadMonitors = function(callback){ + s.beforeMonitorsLoadedOnStartupExtensions.forEach(function(extender){ + extender() + }) + s.systemLog(lang.startUpText4) + //preliminary monitor start + s.knexQuery({ + action: "select", + columns: "*", + table: "Monitors", + },function(err,monitors) { + foundMonitors = monitors + if(err){s.systemLog(err)} + if(monitors && monitors[0]){ + var didNotLoad = 0 + var loadCompleted = 0 + var orphanedVideosForMonitors = {} + var loadMonitor = function(monitor){ + const checkAnother = function(){ + ++loadCompleted + if(monitors[loadCompleted]){ + loadMonitor(monitors[loadCompleted]) + }else{ + if(didNotLoad > 0)console.log(`${didNotLoad} Monitor${didNotLoad === 1 ? '' : 's'} not loaded because Admin user does not exist for them. It may have been deleted.`); + callback() + } + } + if(checkedAdminUsers[monitor.ke]){ + setTimeout(function(){ + if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {} + if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0 + s.initiateMonitorObject(monitor) + s.group[monitor.ke].rawMonitorConfigurations[monitor.mid] = monitor + s.sendMonitorStatus({id:monitor.mid,ke:monitor.ke,status:'Stopped'}); + var monObj = Object.assign(monitor,{id : monitor.mid}) + s.camera(monitor.mode,monObj) + checkAnother() + },1000) }else{ - if(didNotLoad > 0)console.log(`${didNotLoad} Monitor${didNotLoad === 1 ? '' : 's'} not loaded because Admin user does not exist for them. It may have been deleted.`); - callback() + ++didNotLoad + checkAnother() } } - if(checkedAdminUsers[monitor.ke]){ - setTimeout(function(){ - if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {} - if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0 - s.initiateMonitorObject(monitor) - s.group[monitor.ke].rawMonitorConfigurations[monitor.mid] = monitor - s.sendMonitorStatus({id:monitor.mid,ke:monitor.ke,status:'Stopped'}); - var monObj = Object.assign(monitor,{id : monitor.mid}) - s.camera(monitor.mode,monObj) - checkAnother() - },1000) - }else{ - ++didNotLoad - checkAnother() - } + loadMonitor(monitors[loadCompleted]) + }else{ + callback() } - loadMonitor(monitors[loadCompleted]) + }) + } + var checkForOrphanedVideos = function(callback){ + var monitors = foundMonitors + if(monitors && monitors[0]){ + var loadCompleted = 0 + var orphanedVideosForMonitors = {} + var checkForOrphanedVideosForMonitor = function(monitor){ + if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {} + if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0 + s.orphanedVideoCheck(monitor,null,function(orphanedFilesCount){ + if(orphanedFilesCount){ + orphanedVideosForMonitors[monitor.ke][monitor.mid] += orphanedFilesCount + } + ++loadCompleted + if(monitors[loadCompleted]){ + checkForOrphanedVideosForMonitor(monitors[loadCompleted]) + }else{ + s.systemLog(lang.startUpText6+' : '+s.s(orphanedVideosForMonitors)) + delete(foundMonitors) + callback() + } + }) + } + checkForOrphanedVideosForMonitor(monitors[loadCompleted]) }else{ callback() } - }) - } - var checkForOrphanedVideos = function(callback){ - var monitors = foundMonitors - if(monitors && monitors[0]){ - var loadCompleted = 0 - var orphanedVideosForMonitors = {} - var checkForOrphanedVideosForMonitor = function(monitor){ - if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {} - if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0 - s.orphanedVideoCheck(monitor,null,function(orphanedFilesCount){ - if(orphanedFilesCount){ - orphanedVideosForMonitors[monitor.ke][monitor.mid] += orphanedFilesCount - } - ++loadCompleted - if(monitors[loadCompleted]){ - checkForOrphanedVideosForMonitor(monitors[loadCompleted]) - }else{ - s.systemLog(lang.startUpText6+' : '+s.s(orphanedVideosForMonitors)) - delete(foundMonitors) - callback() - } - }) - } - checkForOrphanedVideosForMonitor(monitors[loadCompleted]) - }else{ - callback() } - } - var loadDiskUseForUser = function(user,callback){ - s.systemLog(user.mail+' : '+lang.startUpText0) - var userDetails = JSON.parse(user.details) - s.group[user.ke].sizeLimit = parseFloat(userDetails.size) || 10000 - s.group[user.ke].sizeLimitVideoPercent = parseFloat(userDetails.size_video_percent) || 90 - s.group[user.ke].sizeLimitTimelapseFramesPercent = parseFloat(userDetails.size_timelapse_percent) || 10 - s.knexQuery({ - action: "select", - columns: "*", - table: "Videos", - where: [ - ['ke','=',user.ke], - ['status','!=',0], - ] - },function(err,videos) { + var loadDiskUseForUser = function(user,callback){ + s.systemLog(user.mail+' : '+lang.startUpText0) + var userDetails = JSON.parse(user.details) + s.group[user.ke].sizeLimit = parseFloat(userDetails.size) || 10000 + s.group[user.ke].sizeLimitVideoPercent = parseFloat(userDetails.size_video_percent) || 90 + s.group[user.ke].sizeLimitTimelapseFramesPercent = parseFloat(userDetails.size_timelapse_percent) || 10 s.knexQuery({ action: "select", columns: "*", - table: "Timelapse Frames", - where: [ - ['ke','=',user.ke], - ] - },function(err,timelapseFrames) { - s.knexQuery({ - action: "select", - columns: "*", - table: "Files", - where: [ - ['ke','=',user.ke], - ] - },function(err,files) { - var usedSpaceVideos = 0 - var usedSpaceTimelapseFrames = 0 - var usedSpaceFilebin = 0 - var addStorageData = { - files: [], - videos: [], - timelapeFrames: [], - } - if(videos && videos[0]){ - videos.forEach(function(video){ - video.details = s.parseJSON(video.details) - if(!video.details.dir){ - usedSpaceVideos += video.size - }else{ - addStorageData.videos.push(video) - } - }) - } - if(timelapseFrames && timelapseFrames[0]){ - timelapseFrames.forEach(function(frame){ - frame.details = s.parseJSON(frame.details) - if(!frame.details.dir){ - usedSpaceTimelapseFrames += frame.size - }else{ - addStorageData.timelapeFrames.push(frame) - } - }) - } - if(files && files[0]){ - files.forEach(function(file){ - file.details = s.parseJSON(file.details) - if(!file.details.dir){ - usedSpaceFilebin += file.size - }else{ - addStorageData.files.push(file) - } - }) - } - s.group[user.ke].usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1048576 - s.group[user.ke].usedSpaceVideos = usedSpaceVideos / 1048576 - s.group[user.ke].usedSpaceFilebin = usedSpaceFilebin / 1048576 - s.group[user.ke].usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1048576 - loadAddStorageDiskUseForUser(user,addStorageData,function(){ - callback() - }) - }) - }) - }) - } - var loadCloudDiskUseForUser = function(user,callback){ - var userDetails = JSON.parse(user.details) - user.cloudDiskUse = {} - user.size = 0 - user.limit = userDetails.size - s.cloudDisksLoaded.forEach(function(storageType){ - user.cloudDiskUse[storageType] = { - usedSpace : 0, - firstCount : 0 - } - if(s.cloudDiskUseStartupExtensions[storageType])s.cloudDiskUseStartupExtensions[storageType](user,userDetails) - }) - var loadCloudVideos = function(callback){ - s.knexQuery({ - action: "select", - columns: "*", - table: "Cloud Videos", + table: "Videos", where: [ ['ke','=',user.ke], ['status','!=',0], ] },function(err,videos) { - if(videos && videos[0]){ - videos.forEach(function(video){ - var storageType = JSON.parse(video.details).type - if(!storageType)storageType = 's3' - var videoSize = video.size / 1048576 - user.cloudDiskUse[storageType].usedSpace += videoSize - user.cloudDiskUse[storageType].usedSpaceVideos += videoSize - ++user.cloudDiskUse[storageType].firstCount + s.knexQuery({ + action: "select", + columns: "*", + table: "Timelapse Frames", + where: [ + ['ke','=',user.ke], + ] + },function(err,timelapseFrames) { + s.knexQuery({ + action: "select", + columns: "*", + table: "Files", + where: [ + ['ke','=',user.ke], + ] + },function(err,files) { + var usedSpaceVideos = 0 + var usedSpaceTimelapseFrames = 0 + var usedSpaceFilebin = 0 + var addStorageData = { + files: [], + videos: [], + timelapeFrames: [], + } + if(videos && videos[0]){ + videos.forEach(function(video){ + video.details = s.parseJSON(video.details) + if(!video.details.dir){ + usedSpaceVideos += video.size + }else{ + addStorageData.videos.push(video) + } + }) + } + if(timelapseFrames && timelapseFrames[0]){ + timelapseFrames.forEach(function(frame){ + frame.details = s.parseJSON(frame.details) + if(!frame.details.dir){ + usedSpaceTimelapseFrames += frame.size + }else{ + addStorageData.timelapeFrames.push(frame) + } + }) + } + if(files && files[0]){ + files.forEach(function(file){ + file.details = s.parseJSON(file.details) + if(!file.details.dir){ + usedSpaceFilebin += file.size + }else{ + addStorageData.files.push(file) + } + }) + } + s.group[user.ke].usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1048576 + s.group[user.ke].usedSpaceVideos = usedSpaceVideos / 1048576 + s.group[user.ke].usedSpaceFilebin = usedSpaceFilebin / 1048576 + s.group[user.ke].usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1048576 + loadAddStorageDiskUseForUser(user,addStorageData,function(){ + callback() + }) }) - s.cloudDisksLoaded.forEach(function(storageType){ - var firstCount = user.cloudDiskUse[storageType].firstCount - s.systemLog(user.mail+' : '+lang.startUpText1+' : '+firstCount,storageType,user.cloudDiskUse[storageType].usedSpace) - delete(user.cloudDiskUse[storageType].firstCount) - }) - } - callback() + }) }) } - var loadCloudTimelapseFrames = function(callback){ + var loadCloudDiskUseForUser = function(user,callback){ + var userDetails = JSON.parse(user.details) + user.cloudDiskUse = {} + user.size = 0 + user.limit = userDetails.size + s.cloudDisksLoaded.forEach(function(storageType){ + user.cloudDiskUse[storageType] = { + usedSpace : 0, + firstCount : 0 + } + if(s.cloudDiskUseStartupExtensions[storageType])s.cloudDiskUseStartupExtensions[storageType](user,userDetails) + }) + var loadCloudVideos = function(callback){ + s.knexQuery({ + action: "select", + columns: "*", + table: "Cloud Videos", + where: [ + ['ke','=',user.ke], + ['status','!=',0], + ] + },function(err,videos) { + if(videos && videos[0]){ + videos.forEach(function(video){ + var storageType = JSON.parse(video.details).type + if(!storageType)storageType = 's3' + var videoSize = video.size / 1048576 + user.cloudDiskUse[storageType].usedSpace += videoSize + user.cloudDiskUse[storageType].usedSpaceVideos += videoSize + ++user.cloudDiskUse[storageType].firstCount + }) + s.cloudDisksLoaded.forEach(function(storageType){ + var firstCount = user.cloudDiskUse[storageType].firstCount + s.systemLog(user.mail+' : '+lang.startUpText1+' : '+firstCount,storageType,user.cloudDiskUse[storageType].usedSpace) + delete(user.cloudDiskUse[storageType].firstCount) + }) + } + callback() + }) + } + var loadCloudTimelapseFrames = function(callback){ + s.knexQuery({ + action: "select", + columns: "*", + table: "Cloud Timelapse Frames", + where: [ + ['ke','=',user.ke], + ] + },function(err,frames) { + if(frames && frames[0]){ + frames.forEach(function(frame){ + var storageType = JSON.parse(frame.details).type + if(!storageType)storageType = 's3' + var frameSize = frame.size / 1048576 + user.cloudDiskUse[storageType].usedSpace += frameSize + user.cloudDiskUse[storageType].usedSpaceTimelapseFrames += frameSize + }) + } + callback() + }) + } + loadCloudVideos(function(){ + loadCloudTimelapseFrames(function(){ + s.group[user.ke].cloudDiskUse = user.cloudDiskUse + callback() + }) + }) + } + var loadAddStorageDiskUseForUser = function(user,data,callback){ + var videos = data.videos + var timelapseFrames = data.timelapseFrames + var files = data.files + var userDetails = JSON.parse(user.details) + var userAddStorageData = s.parseJSON(userDetails.addStorage) || {} + var currentStorageNumber = 0 + var readStorageArray = function(){ + var storage = s.listOfStorage[currentStorageNumber] + if(!storage){ + //done all checks, move on to next user + callback() + return + } + var path = storage.value + if(path === ''){ + ++currentStorageNumber + readStorageArray() + return + } + var storageId = path + var storageData = userAddStorageData[storageId] || {} + if(!s.group[user.ke].addStorageUse[storageId])s.group[user.ke].addStorageUse[storageId] = {} + var storageIndex = s.group[user.ke].addStorageUse[storageId] + storageIndex.name = storage.name + storageIndex.path = path + storageIndex.usedSpace = 0 + storageIndex.sizeLimit = parseFloat(storageData.limit) || parseFloat(userDetails.size) || 10000 + var usedSpaceVideos = 0 + var usedSpaceTimelapseFrames = 0 + var usedSpaceFilebin = 0 + if(videos && videos[0]){ + videos.forEach(function(video){ + if(video.details.dir === storage.value){ + usedSpaceVideos += video.size + } + }) + } + if(timelapseFrames && timelapseFrames[0]){ + timelapseFrames.forEach(function(frame){ + if(video.details.dir === storage.value){ + usedSpaceTimelapseFrames += frame.size + } + }) + } + if(files && files[0]){ + files.forEach(function(file){ + if(video.details.dir === storage.value){ + usedSpaceFilebin += file.size + } + }) + } + storageIndex.usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1048576 + storageIndex.usedSpaceVideos = usedSpaceVideos / 1048576 + storageIndex.usedSpaceFilebin = usedSpaceFilebin / 1048576 + storageIndex.usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1048576 + s.systemLog(user.mail+' : '+path+' : '+videos.length,storageIndex.usedSpace) + ++currentStorageNumber + readStorageArray() + } + readStorageArray() + } + var loadAdminUsers = function(callback){ + //get current disk used for each isolated account (admin user) on startup s.knexQuery({ action: "select", columns: "*", - table: "Cloud Timelapse Frames", + table: "Users", where: [ - ['ke','=',user.ke], + ['details','NOT LIKE','%"sub"%'] ] - },function(err,frames) { - if(frames && frames[0]){ - frames.forEach(function(frame){ - var storageType = JSON.parse(frame.details).type - if(!storageType)storageType = 's3' - var frameSize = frame.size / 1048576 - user.cloudDiskUse[storageType].usedSpace += frameSize - user.cloudDiskUse[storageType].usedSpaceTimelapseFrames += frameSize - }) - } - callback() - }) - } - loadCloudVideos(function(){ - loadCloudTimelapseFrames(function(){ - s.group[user.ke].cloudDiskUse = user.cloudDiskUse - callback() - }) - }) - } - var loadAddStorageDiskUseForUser = function(user,data,callback){ - var videos = data.videos - var timelapseFrames = data.timelapseFrames - var files = data.files - var userDetails = JSON.parse(user.details) - var userAddStorageData = s.parseJSON(userDetails.addStorage) || {} - var currentStorageNumber = 0 - var readStorageArray = function(){ - var storage = s.listOfStorage[currentStorageNumber] - if(!storage){ - //done all checks, move on to next user - callback() - return - } - var path = storage.value - if(path === ''){ - ++currentStorageNumber - readStorageArray() - return - } - var storageId = path - var storageData = userAddStorageData[storageId] || {} - if(!s.group[user.ke].addStorageUse[storageId])s.group[user.ke].addStorageUse[storageId] = {} - var storageIndex = s.group[user.ke].addStorageUse[storageId] - storageIndex.name = storage.name - storageIndex.path = path - storageIndex.usedSpace = 0 - storageIndex.sizeLimit = parseFloat(storageData.limit) || parseFloat(userDetails.size) || 10000 - var usedSpaceVideos = 0 - var usedSpaceTimelapseFrames = 0 - var usedSpaceFilebin = 0 - if(videos && videos[0]){ - videos.forEach(function(video){ - if(video.details.dir === storage.value){ - usedSpaceVideos += video.size - } - }) - } - if(timelapseFrames && timelapseFrames[0]){ - timelapseFrames.forEach(function(frame){ - if(video.details.dir === storage.value){ - usedSpaceTimelapseFrames += frame.size - } - }) - } - if(files && files[0]){ - files.forEach(function(file){ - if(video.details.dir === storage.value){ - usedSpaceFilebin += file.size - } - }) - } - storageIndex.usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1048576 - storageIndex.usedSpaceVideos = usedSpaceVideos / 1048576 - storageIndex.usedSpaceFilebin = usedSpaceFilebin / 1048576 - storageIndex.usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1048576 - s.systemLog(user.mail+' : '+path+' : '+videos.length,storageIndex.usedSpace) - ++currentStorageNumber - readStorageArray() - } - readStorageArray() - } - var loadAdminUsers = function(callback){ - //get current disk used for each isolated account (admin user) on startup - s.knexQuery({ - action: "select", - columns: "*", - table: "Users", - where: [ - ['details','NOT LIKE','%"sub"%'] - ] - },function(err,users) { - if(users && users[0]){ - users.forEach(function(user){ - checkedAdminUsers[user.ke] = user - }) - var loadLocalDiskUse = function(callback){ - var count = users.length - var countFinished = 0 + },function(err,users) { + if(users && users[0]){ users.forEach(function(user){ - s.loadGroup(user) - s.loadGroupApps(user) - loadedAccounts.push(user.ke) - loadDiskUseForUser(user,function(){ - ++countFinished - if(countFinished === count){ - callback() - } + checkedAdminUsers[user.ke] = user + }) + var loadLocalDiskUse = function(callback){ + var count = users.length + var countFinished = 0 + users.forEach(function(user){ + s.loadGroup(user) + s.loadGroupApps(user) + loadedAccounts.push(user.ke) + loadDiskUseForUser(user,function(){ + ++countFinished + if(countFinished === count){ + callback() + } + }) + }) + } + var loadCloudDiskUse = function(callback){ + var count = users.length + var countFinished = 0 + users.forEach(function(user){ + loadCloudDiskUseForUser(user,function(){ + ++countFinished + if(countFinished === count){ + callback() + } + }) + }) + } + loadLocalDiskUse(function(){ + loadCloudDiskUse(function(){ + callback() }) }) + }else{ + s.processReady() } - var loadCloudDiskUse = function(callback){ - var count = users.length - var countFinished = 0 - users.forEach(function(user){ - loadCloudDiskUseForUser(user,function(){ - ++countFinished - if(countFinished === count){ - callback() - } - }) - }) - } - loadLocalDiskUse(function(){ - loadCloudDiskUse(function(){ - callback() - }) + }) + } + config.userHasSubscribed = false + var checkSubscription = function(callback){ + var subscriptionFailed = function(){ + console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') + console.error('This Install of Shinobi is NOT Activated') + console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') + console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') + s.systemLog('This Install of Shinobi is NOT Activated') + console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') + console.log('https://licenses.shinobi.video/subscribe') + } + if(config.subscriptionId && config.subscriptionId !== 'sub_XXXXXXXXXXXX'){ + var url = 'https://licenses.shinobi.video/subscribe/check?subscriptionId=' + config.subscriptionId + request(url,{ + method: 'GET', + timeout: 30000 + }, function(err,resp,body){ + var json = s.parseJSON(body) + if(err)console.log(err,json) + var hasSubcribed = json && !!json.ok + config.userHasSubscribed = hasSubcribed + callback(hasSubcribed) + if(config.userHasSubscribed){ + s.systemLog('This Install of Shinobi is Activated') + if(!json.expired){ + s.systemLog(`This License expires on ${json.timeExpires}`) + } + }else{ + subscriptionFailed() + } }) }else{ - s.processReady() + subscriptionFailed() + callback(false) } - }) - } - config.userHasSubscribed = false - var checkSubscription = function(callback){ - var subscriptionFailed = function(){ - console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') - console.error('This Install of Shinobi is NOT Activated') - console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') - console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') - s.systemLog('This Install of Shinobi is NOT Activated') - console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') - console.log('https://licenses.shinobi.video/subscribe') } - if(config.subscriptionId && config.subscriptionId !== 'sub_XXXXXXXXXXXX'){ - var url = 'https://licenses.shinobi.video/subscribe/check?subscriptionId=' + config.subscriptionId - request(url,{ - method: 'GET', - timeout: 30000 - }, function(err,resp,body){ - var json = s.parseJSON(body) - if(err)console.log(err,json) - var hasSubcribed = json && !!json.ok - config.userHasSubscribed = hasSubcribed - callback(hasSubcribed) - if(config.userHasSubscribed){ - s.systemLog('This Install of Shinobi is Activated') - if(!json.expired){ - s.systemLog(`This License expires on ${json.timeExpires}`) - } - }else{ - subscriptionFailed() - } - }) - }else{ - subscriptionFailed() - callback(false) + //check disk space every 20 minutes + if(config.autoDropCache===true){ + setInterval(function(){ + exec('echo 3 > /proc/sys/vm/drop_caches',{detached: true}) + },60000*20) } - } - //check disk space every 20 minutes - if(config.autoDropCache===true){ - setInterval(function(){ - exec('echo 3 > /proc/sys/vm/drop_caches',{detached: true}) - },60000*20) - } - if(config.childNodes.mode !== 'child'){ - //master node - startup functions - //hourly check to see if sizePurge has failed to unlock - //checks to see if request count is the number of monitors + 10 - s.checkForStalePurgeLocks() - //run prerequsite queries, load users and monitors - //sql/database connection with knex - s.databaseEngine = require('knex')(s.databaseOptions) - //run prerequsite queries - s.preQueries() - setTimeout(() => { - //check for subscription - checkSubscription(function(){ - //check terminal commander - checkForTerminalCommands(function(){ - //load administrators (groups) - loadAdminUsers(function(){ - //load monitors (for groups) - loadMonitors(function(){ - //check for orphaned videos - checkForOrphanedVideos(async () => { - s.processReady() + if(config.childNodes.mode !== 'child'){ + //master node - startup functions + //hourly check to see if sizePurge has failed to unlock + //checks to see if request count is the number of monitors + 10 + s.checkForStalePurgeLocks() + //run prerequsite queries, load users and monitors + //sql/database connection with knex + s.databaseEngine = require('knex')(s.databaseOptions) + //run prerequsite queries + s.preQueries() + setTimeout(() => { + //check for subscription + checkSubscription(function(){ + //check terminal commander + checkForTerminalCommands(function(){ + //load administrators (groups) + loadAdminUsers(function(){ + //load monitors (for groups) + loadMonitors(function(){ + //check for orphaned videos + checkForOrphanedVideos(async () => { + s.processReady() + }) }) }) }) }) - }) - },1500) - } + },1500) + } + }) } diff --git a/web/libs/js/dash2.socketio.js b/web/libs/js/dash2.socketio.js index a1406e46..fcf809c9 100644 --- a/web/libs/js/dash2.socketio.js +++ b/web/libs/js/dash2.socketio.js @@ -302,7 +302,6 @@ $.ccio.globalWebsocket=function(d,user){ uid: user.uid, ke: d.ke, id: d.id, - subscriptionId: subscriptionId, // channel: channel }) if(!$.ccio.mon[d.ke+d.id+user.auth_token].ctx||$.ccio.mon[d.ke+d.id+user.auth_token].ctx.length===0){ @@ -703,15 +702,15 @@ $user.ws.on('connect',function (d){ $(document).ready(function(e){ $.ccio.init('id',$user); if(location.search === '?assemble=1'){ - $user.ws.emit('initUser',{ - subscriptionId: subscriptionId, + $user.ws.emit('p2pInitUser',{ user: { ke: $user.ke, mail: $user.mail, auth_token: $user.auth_token, details: $user.details, uid: $user.uid, - } + }, + machineId: machineId }) }else{ $.ccio.cx({f:'init',ke:$user.ke,auth:$user.auth_token,uid:$user.uid}) diff --git a/web/pages/blocks/header.ejs b/web/pages/blocks/header.ejs index 7273598e..d31563ce 100644 --- a/web/pages/blocks/header.ejs +++ b/web/pages/blocks/header.ejs @@ -22,4 +22,4 @@ if(!string){string=''} return string.replace(/'/g,"\\'") }%> - + From 5dd153f288e398211950f514f931ee4f7ea19214 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Sun, 20 Sep 2020 23:11:07 -0700 Subject: [PATCH 2/5] clean p2p logging --- libs/commander.js | 19 ++++++++++--------- libs/socketio.js | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libs/commander.js b/libs/commander.js index b05d545d..6068a438 100644 --- a/libs/commander.js +++ b/libs/commander.js @@ -48,17 +48,17 @@ module.exports = function(s,config,lang,app,io){ } // const createSocketConnections = () => { - console.log(`Connecting to ${config.p2pHost}`) + s.systemLog('p2p',`Connecting to ${config.p2pHost}...`) const connectionToP2PServer = socketIOClient(config.p2pHost, {transports:['websocket']}); if(!config.p2pApiKey){ - console.log(`Please fill 'p2pApiKey' in your conf.json.`) + s.systemLog('p2p',`Please fill 'p2pApiKey' in your conf.json.`) } if(!config.p2pGroupId){ - console.log(`Please fill 'p2pGroupId' in your conf.json.`) + s.systemLog('p2p',`Please fill 'p2pGroupId' in your conf.json.`) } config.machineId = config.machineId || s.gid(20) connectionToP2PServer.on('connect',function(){ - console.log(`Connected ${config.p2pHost}`) + s.systemLog('p2p',`Connected ${config.p2pHost}!`) connectionToP2PServer.emit('initMachine',{ port: config.port, apiKey: config.p2pApiKey, @@ -106,11 +106,11 @@ module.exports = function(s,config,lang,app,io){ }) connectionToP2PServer.on('wsInit',function(rawRequest){ - console.log('wsInit') + s.systemLog('p2pWsInit',rawRequest) var user = rawRequest.user var clientConnectionToMachine = createShinobiSocketConnection(rawRequest.cnid) - connectionToP2PServer.on('f',function(rawRequest){ - clientConnectionToMachine.emit('f',rawRequest.data) + connectionToP2PServer.on('f',function(data){ + clientConnectionToMachine.emit('f',data) }) clientConnectionToMachine.on('connect',function(){ clientConnectionToMachine.emit('f',{ @@ -133,11 +133,12 @@ module.exports = function(s,config,lang,app,io){ connectionToP2PServer.on('allowDisconnect',function(bool){ connectionToP2PServer.allowDisconnect = true; connectionToP2PServer.disconnect() - console.log('Server said to go away') + s.systemLog('p2p','Server Forced Disconnection') }); const onDisconnect = () => { - console.log('disconnected p2p') + s.systemLog('p2p','Disconnected') if(!connectionToP2PServer.allowDisconnect){ + s.systemLog('p2p','Attempting Reconnection...') setTimeout(function(){ connectionToP2PServer.connect() },3000) diff --git a/libs/socketio.js b/libs/socketio.js index 731e4c7a..77f0a058 100644 --- a/libs/socketio.js +++ b/libs/socketio.js @@ -393,7 +393,6 @@ module.exports = function(s,config,lang,io){ extender(r,cn,d,tx) }) } - console.log(cn.ip) streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail) return; } From fe928efcabab624b73d90fe9d5fbf1481333cf2d Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Mon, 21 Sep 2020 23:49:18 -0700 Subject: [PATCH 3/5] minor cleanup to usage --- libs/commander.js | 127 +++++++++++++++++++++++++--------- libs/monitor.js | 2 +- libs/webServer.js | 2 + web/libs/js/dash2.config.js | 2 +- web/libs/js/dash2.socketio.js | 6 +- web/pages/admin.ejs | 6 +- web/pages/super.ejs | 6 +- 7 files changed, 108 insertions(+), 43 deletions(-) diff --git a/libs/commander.js b/libs/commander.js index 6068a438..259c9f57 100644 --- a/libs/commander.js +++ b/libs/commander.js @@ -2,12 +2,13 @@ module.exports = function(s,config,lang,app,io){ // Shinobi P2P Client Script if(config.p2pEnabled){ if(!config.p2pHost)config.p2pHost = 'ws://163.172.180.205:8084' - var p2pClientConnectionStaticName = 'Commander' - var p2pClientConnections = {} - var runningRequests = {} - var request = require('request'); + const p2pClientConnectionStaticName = 'Commander' + const p2pClientConnections = {} + const runningRequests = {} + const connectedUserWebSockets = {} + const request = require('request'); const socketIOClient = require('socket.io-client'); - var parseJSON = function(string){ + const parseJSON = function(string){ var parsed = string try{ parsed = JSON.parse(string) @@ -16,7 +17,7 @@ module.exports = function(s,config,lang,app,io){ } return parsed } - var createQueryStringFromObject = function(obj){ + const createQueryStringFromObject = function(obj){ var queryString = '' var keys = Object.keys(obj) keys.forEach(function(key){ @@ -25,8 +26,8 @@ module.exports = function(s,config,lang,app,io){ }) return queryString } - var doRequest = function(url,method,data,callback,onDataReceived){ - var requestEndpoint = url + const doRequest = function(url,method,data,callback,onDataReceived){ + var requestEndpoint = `${config.sslEnabled ? `https` : 'http'}://localhost:${config.sslEnabled ? config.ssl.port : config.port}` + url if(method === 'GET' && data){ requestEndpoint += '?' + createQueryStringFromObject(data) } @@ -41,14 +42,14 @@ module.exports = function(s,config,lang,app,io){ onDataReceived(data) }) } - var createShinobiSocketConnection = function(connectionId){ - var masterConnectionToMachine = socketIOClient(`ws://localhost:${config.port}`, {transports:['websocket']}) + const createShinobiSocketConnection = (connectionId) => { + const masterConnectionToMachine = socketIOClient(`ws://localhost:${config.port}`, {transports:['websocket']}) p2pClientConnections[connectionId || p2pClientConnectionStaticName] = masterConnectionToMachine return masterConnectionToMachine } // const createSocketConnections = () => { - s.systemLog('p2p',`Connecting to ${config.p2pHost}...`) + s.debugLog('p2p',`Connecting to ${config.p2pHost}...`) const connectionToP2PServer = socketIOClient(config.p2pHost, {transports:['websocket']}); if(!config.p2pApiKey){ s.systemLog('p2p',`Please fill 'p2pApiKey' in your conf.json.`) @@ -57,7 +58,7 @@ module.exports = function(s,config,lang,app,io){ s.systemLog('p2p',`Please fill 'p2pGroupId' in your conf.json.`) } config.machineId = config.machineId || s.gid(20) - connectionToP2PServer.on('connect',function(){ + connectionToP2PServer.on('connect', () => { s.systemLog('p2p',`Connected ${config.p2pHost}!`) connectionToP2PServer.emit('initMachine',{ port: config.port, @@ -69,12 +70,13 @@ module.exports = function(s,config,lang,app,io){ subscriptionId: config.subscriptionId || 'notActivated' }) }) - connectionToP2PServer.on('httpClose',function(requestId){ + connectionToP2PServer.on('httpClose',(requestId) => { if(runningRequests[requestId] && runningRequests[requestId].abort){ runningRequests[requestId].abort() + delete(runningRequests[requestId]) } }) - connectionToP2PServer.on('http',function(rawRequest){ + connectionToP2PServer.on('http',(rawRequest) => { runningRequests[rawRequest.rid] = doRequest( rawRequest.url, rawRequest.method, @@ -86,7 +88,7 @@ module.exports = function(s,config,lang,app,io){ rid: rawRequest.rid }) }, - function(data){ + (data) => { if(!rawRequest.bodyOnEnd)connectionToP2PServer.emit('httpResponseChunk',{ data: data, rid: rawRequest.rid @@ -94,52 +96,113 @@ module.exports = function(s,config,lang,app,io){ }) }) const masterConnectionToMachine = createShinobiSocketConnection() - masterConnectionToMachine.on('connect',function(){ + masterConnectionToMachine.on('connect', () => { masterConnectionToMachine.emit('f',{ f: 'init', + auth: s.gid(30), ke: config.p2pTargetGroupId, uid: config.p2pTargetUserId }) }) - masterConnectionToMachine.on('f',function(data){ + masterConnectionToMachine.on('f',(data) => { connectionToP2PServer.emit('f',data) }) - connectionToP2PServer.on('wsInit',function(rawRequest){ - s.systemLog('p2pWsInit',rawRequest) - var user = rawRequest.user - var clientConnectionToMachine = createShinobiSocketConnection(rawRequest.cnid) - connectionToP2PServer.on('f',function(data){ - clientConnectionToMachine.emit('f',data) - }) - clientConnectionToMachine.on('connect',function(){ + connectionToP2PServer.on('wsInit',(rawRequest) => { + s.debugLog('p2pWsInit',rawRequest) + const user = rawRequest.user + const clientConnectionToMachine = createShinobiSocketConnection(rawRequest.cnid) + connectedUserWebSockets[user.auth_token] = user; + clientConnectionToMachine.on('connect', () => { + s.debugLog('init',user.auth_token) clientConnectionToMachine.emit('f',{ f: 'init', + auth: user.auth_token, ke: user.ke, uid: user.uid, }) - }) - clientConnectionToMachine.on('f',function(data){ - connectionToP2PServer.emit('f',{data: data, cnid: rawRequest.cnid}) + }); + ([ + 'f', + ]).forEach((target) => { + connectionToP2PServer.on(target,(data) => { + clientConnectionToMachine.emit(target,data) + }) + clientConnectionToMachine.on(target,(data) => { + connectionToP2PServer.emit(target,{data: data, cnid: rawRequest.cnid}) + }) }) }); - connectionToP2PServer.on('wsDestroy',function(rawRequest){ + ([ + 'a', + 'r', + 'gps', + 'e', + 'super', + ]).forEach((target) => { + connectionToP2PServer.on(target,(data) => { + var clientConnectionToMachine + if(data.f === 'init'){ + clientConnectionToMachine = createShinobiSocketConnection(data.cnid) + clientConnectionToMachine.on('connect', () => { + clientConnectionToMachine.on(target,(fromData) => { + connectionToP2PServer.emit(target,{data: fromData, cnid: data.cnid}) + }) + clientConnectionToMachine.on('f',(fromData) => { + connectionToP2PServer.emit('f',{data: fromData, cnid: data.cnid}) + }) + clientConnectionToMachine.emit(target,data) + }); + }else{ + clientConnectionToMachine = p2pClientConnections[data.cnid] + clientConnectionToMachine.emit(target,data) + } + }) + + }); + ([ + 'h265', + 'Base64', + 'FLV', + 'MP4', + ]).forEach((target) => { + connectionToP2PServer.on(target,(initData) => { + if(connectedUserWebSockets[initData.auth]){ + const clientConnectionToMachine = createShinobiSocketConnection(initData.auth + initData.ke + initData.id) + clientConnectionToMachine.on('connect', () => { + clientConnectionToMachine.emit(target,initData) + }); + clientConnectionToMachine.on('data',(data) => { + connectionToP2PServer.emit('data',{data: data, cnid: initData.cnid}) + }); + }else{ + s.debugLog('disconnect now!') + } + }) + }); + connectionToP2PServer.on('wsDestroyStream',(clientKey) => { + if(p2pClientConnections[clientKey]){ + p2pClientConnections[clientKey].disconnect(); + } + delete(p2pClientConnections[clientKey]) + }); + connectionToP2PServer.on('wsDestroy',(rawRequest) => { if(p2pClientConnections[rawRequest.cnid]){ p2pClientConnections[rawRequest.cnid].disconnect(); } delete(p2pClientConnections[rawRequest.cnid]) }); - connectionToP2PServer.on('allowDisconnect',function(bool){ + connectionToP2PServer.on('allowDisconnect',(bool) => { connectionToP2PServer.allowDisconnect = true; connectionToP2PServer.disconnect() - s.systemLog('p2p','Server Forced Disconnection') + s.debugLog('p2p','Server Forced Disconnection') }); const onDisconnect = () => { s.systemLog('p2p','Disconnected') if(!connectionToP2PServer.allowDisconnect){ s.systemLog('p2p','Attempting Reconnection...') - setTimeout(function(){ + setTimeout(() => { connectionToP2PServer.connect() },3000) } diff --git a/libs/monitor.js b/libs/monitor.js index e28d61b3..80cb56b0 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -455,7 +455,7 @@ module.exports = function(s,config,lang){ ke: e.ke },'GRP_'+e.ke) }else{ - console.log('not image') + s.debugLog('Damaged Snapshot Data') s.tx({f:'monitor_snapshot',snapshot:e.mon.name,snapshot_format:'plc',mid:e.mid,ke:e.ke},'GRP_'+e.ke) } }else{ diff --git a/libs/webServer.js b/libs/webServer.js index 15803a9a..a335c955 100644 --- a/libs/webServer.js +++ b/libs/webServer.js @@ -57,7 +57,9 @@ module.exports = function(s,config,lang,io){ //SSL options var wellKnownDirectory = s.mainDirectory + '/web/.well-known' if(fs.existsSync(wellKnownDirectory))app.use('/.well-known',express.static(wellKnownDirectory)) + config.sslEnabled = false if(config.ssl&&config.ssl.key&&config.ssl.cert){ + config.sslEnabled = true config.ssl.key=fs.readFileSync(s.checkRelativePath(config.ssl.key),'utf8') config.ssl.cert=fs.readFileSync(s.checkRelativePath(config.ssl.cert),'utf8') if(config.ssl.port === undefined){ diff --git a/web/libs/js/dash2.config.js b/web/libs/js/dash2.config.js index c7a42377..4c0f00fc 100644 --- a/web/libs/js/dash2.config.js +++ b/web/libs/js/dash2.config.js @@ -3,7 +3,7 @@ $.ccio={ mon:{}, useUTC: <%- config.useUTC || false %>, definitions: <%-JSON.stringify(define)%>, - libURL: location.search === '?assemble=1' ? location.pathname + '/' : '<%-window.libURL%>', + libURL: location.search === '?p2p=1' ? location.pathname + '/' : '<%-window.libURL%>', isAppleDevice: navigator.userAgent.match(/(iPod|iPhone|iPad)/)||(navigator.userAgent.match(/(Safari)/)&&!navigator.userAgent.match('Chrome')) }; <% if(config.DropboxAppKey){ %> diff --git a/web/libs/js/dash2.socketio.js b/web/libs/js/dash2.socketio.js index fcf809c9..cf17a902 100644 --- a/web/libs/js/dash2.socketio.js +++ b/web/libs/js/dash2.socketio.js @@ -277,7 +277,7 @@ $.ccio.globalWebsocket=function(d,user){ if($.ccio.op().jpeg_on===true){ $.ccio.init('jpegMode',$.ccio.mon[d.ke+d.id+user.auth_token]); }else{ - if(location.search === '?assemble=1'){ + if(location.search === '?p2p=1'){ var path = '/socket.io' }else{ var path = tool.checkCorrectPathEnding(location.pathname)+'socket.io' @@ -689,7 +689,7 @@ $.ccio.globalWebsocket=function(d,user){ break; } } -if(location.search === '?assemble=1'){ +if(location.search === '?p2p=1'){ $user.ws=io(location.origin,{ path : '/socket.io' }); @@ -701,7 +701,7 @@ if(location.search === '?assemble=1'){ $user.ws.on('connect',function (d){ $(document).ready(function(e){ $.ccio.init('id',$user); - if(location.search === '?assemble=1'){ + if(location.search === '?p2p=1'){ $user.ws.emit('p2pInitUser',{ user: { ke: $user.ke, diff --git a/web/pages/admin.ejs b/web/pages/admin.ejs index c009ac34..66f45b91 100644 --- a/web/pages/admin.ejs +++ b/web/pages/admin.ejs @@ -86,7 +86,7 @@ <% include blocks/confirm.ejs %> <% include blocks/subpermissions.ejs %> @@ -104,7 +104,7 @@ <% customAutoLoad.superLibsCss.forEach(function(lib){ %> @@ -173,7 +173,7 @@ switch($user.lang){ }) break; } -if(location.search === '?assemble=1'){ +if(location.search === '?p2p=1'){ $.ccio.ws=io(location.origin,{ path : '/socket.io' }); @@ -184,7 +184,7 @@ if(location.search === '?assemble=1'){ } $.ccio.cx=function(x){return $.ccio.ws.emit('super',x)} $.ccio.ws.on('connect',function(d){ - $.ccio.cx({f:'init',mail:$user.mail,pass:$user.pass}) + $.ccio.cx({f:'init',mail:$user.mail,pass:$user.pass,machineId: `<%- config.machineId %>`}) }) $.ccio.ws.on('f',function(d){ switch(d.f){ From 767e059adb1bdc7545138c2226dde05cf98d7208 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Mon, 21 Sep 2020 23:58:42 -0700 Subject: [PATCH 4/5] fix web paths in super and admin pages --- web/pages/admin.ejs | 6 +++--- web/pages/blocks/mainpermissions.ejs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/web/pages/admin.ejs b/web/pages/admin.ejs index 66f45b91..86cdba98 100644 --- a/web/pages/admin.ejs +++ b/web/pages/admin.ejs @@ -179,7 +179,7 @@ $.aN.e.submit(function(e){ e.preventDefault(); e.s = $.aN.e.serializeObject() e.m = $('#msg').empty() - $.post('<%=originalURL%><%=config.webPaths.adminApiPrefix%>'+$user.auth_token+'/accounts/'+$user.ke+'/register',{data:e.s},function(d){ + $.post(adminApiPrefix + $user.auth_token+'/accounts/'+$user.ke+'/register',{data:e.s},function(d){ if(d.msg){ e.m.text(d.msg) } @@ -197,7 +197,7 @@ $.sU.e.on('click','.delete',function(e){ var html = 'Do you want to delete '+subAccountEmail+'? You cannot recover this account.' $.confirm.body.html(html) $.confirm.click({title:'Delete',class:'btn-danger'},function(){ - $.post('<%=originalURL%><%=config.webPaths.adminApiPrefix%>'+$user.auth_token+'/accounts/'+$user.ke+'/delete',{ + $.post(adminApiPrefix + $user.auth_token+'/accounts/'+$user.ke+'/delete',{ uid: subAccountUid, mail: subAccountEmail },function(data){}) @@ -269,7 +269,7 @@ $.pR.e.on('change','[monitor]',function(e){ $.pR.f.submit(function(e){ e.preventDefault() var form = $(this).serializeObject() - $.post('<%=originalURL%><%=config.webPaths.adminApiPrefix%>'+$user.auth_token+'/accounts/'+$user.ke+'/edit',{ + $.post(adminApiPrefix + $user.auth_token+'/accounts/'+$user.ke+'/edit',{ uid: $.pR.user, mail: $.ccio.subs[$.pR.user].mail, data: form diff --git a/web/pages/blocks/mainpermissions.ejs b/web/pages/blocks/mainpermissions.ejs index 7fbbd330..bea6c7f9 100644 --- a/web/pages/blocks/mainpermissions.ejs +++ b/web/pages/blocks/mainpermissions.ejs @@ -213,7 +213,7 @@ $.aN.f.submit(function(e){ webPath = 'editAdmin' postData.account = $.aN.selected } - $.post('<%=originalURL%><%=config.webPaths.superApiPrefix%>'+$user.sessionKey+'/accounts/'+webPath,postData,function(data){ + $.post(superApiPrefix + $user.sessionKey+'/accounts/'+webPath,postData,function(data){ console.log(data) if(data.ok === true){ $.aN.e.modal('hide') @@ -295,7 +295,7 @@ $.aC.e.on('click','.delete',function(e){ e.html='Do you want to delete '+e.account.mail+'? You cannot recover this account. Files will remain in the filesystem. If you choose to create an account with the same Group Key it will have the previous events activated in that account.' $.confirm.body.html(e.html) $.confirm.click({title:'Delete',class:'btn-danger'},function(){ - $.post('<%=originalURL%><%=config.webPaths.superApiPrefix%>'+$user.sessionKey+'/accounts/deleteAdmin',{ + $.post(superApiPrefix + $user.sessionKey+'/accounts/deleteAdmin',{ account : e.account, // "deleteSubAccounts" : "1", // "deleteMonitors" : "1", From 70d22322e1e3307a1b8a6cdd3d764ef9b67cf28a Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Tue, 22 Sep 2020 10:21:08 -0700 Subject: [PATCH 5/5] P2P : Machine ID is now forced as P2P API Key and Group Key --- libs/commander.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/commander.js b/libs/commander.js index 259c9f57..bb62c56e 100644 --- a/libs/commander.js +++ b/libs/commander.js @@ -57,7 +57,7 @@ module.exports = function(s,config,lang,app,io){ if(!config.p2pGroupId){ s.systemLog('p2p',`Please fill 'p2pGroupId' in your conf.json.`) } - config.machineId = config.machineId || s.gid(20) + config.machineId = config.p2pApiKey + '' + config.p2pGroupId connectionToP2PServer.on('connect', () => { s.systemLog('p2p',`Connected ${config.p2pHost}!`) connectionToP2PServer.emit('initMachine',{ @@ -66,7 +66,6 @@ module.exports = function(s,config,lang,app,io){ groupId: config.p2pGroupId, targetUserId: config.p2pTargetUserId, targetGroupId: config.p2pTargetGroupId, - machineId: config.machineId, subscriptionId: config.subscriptionId || 'notActivated' }) })