ShinobiSystems
/
Shinobi
mirror of https://gitlab.com/Shinobi-Systems/Shinobi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'p2p-framework' into 'dev' 

P2P Service Connector (Optional)

See merge request Shinobi-Systems/Shinobi!239
fix-non-showing-inputs
Moe 2020-09-22 17:27:30 +00:00
parent 7ce737395c 70d22322e1
commit bec8039b79
12 changed files with 674 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
camera.js
View File

@ -26,7 +26,7 @@ require('./libs/codeTester.js')(s,config,lang)
//get version
require('./libs/version.js')(s,config,lang)
//video processing engine
require('./libs/ffmpeg.js')(s,config,lang,function(ffmpeg){
require('./libs/ffmpeg.js')(s,config,lang,async function(ffmpeg){
//ffmpeg coProcessor
require('./libs/ffmpegCoProcessor.js')(s,config,lang,ffmpeg)
//database connection : mysql, sqlite3..
@ -86,5 +86,7 @@ require('./libs/ffmpeg.js')(s,config,lang,function(ffmpeg){
//scheduling engine
require('./libs/scheduler.js')(s,config,lang,app,io)
//on-start actions, daemon(s) starter
require('./libs/startup.js')(s,config,lang)
await require('./libs/startup.js')(s,config,lang)
//p2p, commander
require('./libs/commander.js')(s,config,lang)
})

229
libs/commander.js Normal file
View File

@ -0,0 +1,229 @@
module.exports = function(s,config,lang,app,io){
// Shinobi P2P Client Script
if(config.p2pEnabled){
if(!config.p2pHost)config.p2pHost = 'ws://163.172.180.205:8084'
const p2pClientConnectionStaticName = 'Commander'
const p2pClientConnections = {}
const runningRequests = {}
const connectedUserWebSockets = {}
const request = require('request');
const socketIOClient = require('socket.io-client');
const parseJSON = function(string){
var parsed = string
try{
parsed = JSON.parse(string)
}catch(err){
}
return parsed
}
const createQueryStringFromObject = function(obj){
var queryString = ''
var keys = Object.keys(obj)
keys.forEach(function(key){
var value = obj[key]
queryString += `&${key}=${value}`
})
return queryString
}
const doRequest = function(url,method,data,callback,onDataReceived){
var requestEndpoint = `${config.sslEnabled ? `https` : 'http'}://localhost:${config.sslEnabled ? config.ssl.port : config.port}` + url
if(method === 'GET' && data){
requestEndpoint += '?' + createQueryStringFromObject(data)
}
return request(requestEndpoint,{
method: method,
json: method !== 'GET' ? (data ? data : null) : null
}, function(err,resp,body){
// var json = parseJSON(body)
if(err)console.error(err,data)
callback(err,body,resp)
}).on('data', function(data) {
onDataReceived(data)
})
}
const createShinobiSocketConnection = (connectionId) => {
const masterConnectionToMachine = socketIOClient(`ws://localhost:${config.port}`, {transports:['websocket']})
p2pClientConnections[connectionId || p2pClientConnectionStaticName] = masterConnectionToMachine
return masterConnectionToMachine
}
//
const createSocketConnections = () => {
s.debugLog('p2p',`Connecting to ${config.p2pHost}...`)
const connectionToP2PServer = socketIOClient(config.p2pHost, {transports:['websocket']});
if(!config.p2pApiKey){
s.systemLog('p2p',`Please fill 'p2pApiKey' in your conf.json.`)
}
if(!config.p2pGroupId){
s.systemLog('p2p',`Please fill 'p2pGroupId' in your conf.json.`)
}
config.machineId = config.p2pApiKey + '' + config.p2pGroupId
connectionToP2PServer.on('connect', () => {
s.systemLog('p2p',`Connected ${config.p2pHost}!`)
connectionToP2PServer.emit('initMachine',{
port: config.port,
apiKey: config.p2pApiKey,
groupId: config.p2pGroupId,
targetUserId: config.p2pTargetUserId,
targetGroupId: config.p2pTargetGroupId,
subscriptionId: config.subscriptionId || 'notActivated'
})
})
connectionToP2PServer.on('httpClose',(requestId) => {
if(runningRequests[requestId] && runningRequests[requestId].abort){
runningRequests[requestId].abort()
delete(runningRequests[requestId])
}
})
connectionToP2PServer.on('http',(rawRequest) => {
runningRequests[rawRequest.rid] = doRequest(
rawRequest.url,
rawRequest.method,
rawRequest.data,
function(err,json,resp){
connectionToP2PServer.emit('httpResponse',{
err: err,
json: rawRequest.bodyOnEnd ? json : null,
rid: rawRequest.rid
})
},
(data) => {
if(!rawRequest.bodyOnEnd)connectionToP2PServer.emit('httpResponseChunk',{
data: data,
rid: rawRequest.rid
})
})
})
const masterConnectionToMachine = createShinobiSocketConnection()
masterConnectionToMachine.on('connect', () => {
masterConnectionToMachine.emit('f',{
f: 'init',
auth: s.gid(30),
ke: config.p2pTargetGroupId,
uid: config.p2pTargetUserId
})
})
masterConnectionToMachine.on('f',(data) => {
connectionToP2PServer.emit('f',data)
})
connectionToP2PServer.on('wsInit',(rawRequest) => {
s.debugLog('p2pWsInit',rawRequest)
const user = rawRequest.user
const clientConnectionToMachine = createShinobiSocketConnection(rawRequest.cnid)
connectedUserWebSockets[user.auth_token] = user;
clientConnectionToMachine.on('connect', () => {
s.debugLog('init',user.auth_token)
clientConnectionToMachine.emit('f',{
f: 'init',
auth: user.auth_token,
ke: user.ke,
uid: user.uid,
})
});
([
'f',
]).forEach((target) => {
connectionToP2PServer.on(target,(data) => {
clientConnectionToMachine.emit(target,data)
})
clientConnectionToMachine.on(target,(data) => {
connectionToP2PServer.emit(target,{data: data, cnid: rawRequest.cnid})
})
})
});
([
'a',
'r',
'gps',
'e',
'super',
]).forEach((target) => {
connectionToP2PServer.on(target,(data) => {
var clientConnectionToMachine
if(data.f === 'init'){
clientConnectionToMachine = createShinobiSocketConnection(data.cnid)
clientConnectionToMachine.on('connect', () => {
clientConnectionToMachine.on(target,(fromData) => {
connectionToP2PServer.emit(target,{data: fromData, cnid: data.cnid})
})
clientConnectionToMachine.on('f',(fromData) => {
connectionToP2PServer.emit('f',{data: fromData, cnid: data.cnid})
})
clientConnectionToMachine.emit(target,data)
});
}else{
clientConnectionToMachine = p2pClientConnections[data.cnid]
clientConnectionToMachine.emit(target,data)
}
})
});
([
'h265',
'Base64',
'FLV',
'MP4',
]).forEach((target) => {
connectionToP2PServer.on(target,(initData) => {
if(connectedUserWebSockets[initData.auth]){
const clientConnectionToMachine = createShinobiSocketConnection(initData.auth + initData.ke + initData.id)
clientConnectionToMachine.on('connect', () => {
clientConnectionToMachine.emit(target,initData)
});
clientConnectionToMachine.on('data',(data) => {
connectionToP2PServer.emit('data',{data: data, cnid: initData.cnid})
});
}else{
s.debugLog('disconnect now!')
}
})
});
connectionToP2PServer.on('wsDestroyStream',(clientKey) => {
if(p2pClientConnections[clientKey]){
p2pClientConnections[clientKey].disconnect();
}
delete(p2pClientConnections[clientKey])
});
connectionToP2PServer.on('wsDestroy',(rawRequest) => {
if(p2pClientConnections[rawRequest.cnid]){
p2pClientConnections[rawRequest.cnid].disconnect();
}
delete(p2pClientConnections[rawRequest.cnid])
});
connectionToP2PServer.on('allowDisconnect',(bool) => {
connectionToP2PServer.allowDisconnect = true;
connectionToP2PServer.disconnect()
s.debugLog('p2p','Server Forced Disconnection')
});
const onDisconnect = () => {
s.systemLog('p2p','Disconnected')
if(!connectionToP2PServer.allowDisconnect){
s.systemLog('p2p','Attempting Reconnection...')
setTimeout(() => {
connectionToP2PServer.connect()
},3000)
}
}
connectionToP2PServer.on('error',onDisconnect)
connectionToP2PServer.on('disconnect',onDisconnect)
}
if(config.p2pTargetGroupId && config.p2pTargetUserId){
createSocketConnections()
}else{
s.knexQuery({
action: "select",
columns: "ke,uid",
table: "Users",
where: [],
limit: 1
},(err,r) => {
const firstUser = r[0]
config.p2pTargetUserId = firstUser.uid
config.p2pTargetGroupId = firstUser.ke
createSocketConnections()
})
}
}
}

1
libs/monitor.js
View File

@ -458,6 +458,7 @@ module.exports = function(s,config,lang){
ke: e.ke
},'GRP_'+e.ke)
}else{
s.debugLog('Damaged Snapshot Data')
s.tx({f:'monitor_snapshot',snapshot:e.mon.name,snapshot_format:'plc',mid:e.mid,ke:e.ke},'GRP_'+e.ke)
}
}else{

32
libs/socketio.js
View File

@ -54,17 +54,21 @@ module.exports = function(s,config,lang,io){
}
}
const streamConnectionAuthentication = (options) => {
const streamConnectionAuthentication = (options,ipAddress) => {
return new Promise( (resolve,reject) => {
var isInternal = false
if(ipAddress.indexOf('localhost') > -1 || ipAddress.indexOf('127.0.0.1') > -1){
isInternal = true
}
const baseWheres = [
['ke','=',options.ke],
['uid','=',options.uid],
]
s.knexQuery({
action: "select",
columns: "ke,uid,auth,mail,details",
table: "Users",
where: [
['ke','=',options.ke],
['auth','=',options.auth],
['uid','=',options.uid],
]
where: baseWheres.concat(!isInternal ? [['auth','=',options.auth]] : [])
},(err,r) => {
if(r&&r[0]){
resolve(r)
@ -73,11 +77,7 @@ module.exports = function(s,config,lang,io){
action: "select",
columns: "*",
table: "API",
where: [
['ke','=',options.ke],
['code','=',options.auth],
['uid','=',options.uid],
]
where: baseWheres.concat(!isInternal ? [['code','=',options.auth]] : [])
},(err,r) => {
if(r && r[0]){
r = r[0]
@ -168,7 +168,7 @@ module.exports = function(s,config,lang,io){
if(s.group[d.ke]&&s.group[d.ke].users&&s.group[d.ke].users[d.auth]){
onSuccess(s.group[d.ke].users[d.auth]);
}else{
streamConnectionAuthentication(d).then(onSuccess).catch(onFail)
streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail)
}
})
//unique Base64 socket stream
@ -201,7 +201,7 @@ module.exports = function(s,config,lang,io){
if(s.group[d.ke]&&s.group[d.ke].users&&s.group[d.ke].users[d.auth]){
onSuccess(s.group[d.ke].users[d.auth]);
}else{
streamConnectionAuthentication(d).then(onSuccess).catch(onFail)
streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail)
}
})
//unique FLV socket stream
@ -234,7 +234,7 @@ module.exports = function(s,config,lang,io){
if(s.group[d.ke] && s.group[d.ke].users && s.group[d.ke].users[d.auth]){
onSuccess(s.group[d.ke].users[d.auth]);
}else{
streamConnectionAuthentication(d).then(onSuccess).catch(onFail)
streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail)
}
})
//unique MP4 socket stream
@ -315,7 +315,7 @@ module.exports = function(s,config,lang,io){
if(s.group[d.ke]&&s.group[d.ke].users&&s.group[d.ke].users[d.auth]){
onSuccess(s.group[d.ke].users[d.auth]);
}else{
streamConnectionAuthentication(d).then(onSuccess).catch(onFail)
streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail)
}
})
//main socket control functions
@ -396,7 +396,7 @@ module.exports = function(s,config,lang,io){
extender(r,cn,d,tx)
})
}
streamConnectionAuthentication(d).then(onSuccess).catch(onFail)
streamConnectionAuthentication(d,cn.ip).then(onSuccess).catch(onFail)
return;
}
if((d.id||d.uid||d.mid)&&cn.ke){

803
libs/startup.js
View File

@ -6,443 +6,446 @@ var crypto = require('crypto');
var exec = require('child_process').exec;
var execSync = require('child_process').execSync;
module.exports = function(s,config,lang,io){
var checkedAdminUsers = {}
console.log('FFmpeg version : '+s.ffmpegVersion)
console.log('Node.js version : '+process.version)
s.processReady = function(){
delete(checkedAdminUsers)
s.systemLog(lang.startUpText5)
s.onProcessReadyExtensions.forEach(function(extender){
extender(true)
})
process.send('ready')
}
var checkForTerminalCommands = function(callback){
var next = function(){
if(callback)callback()
}
if(!s.isWin && s.packageJson.mainDirectory !== '.'){
var etcPath = '/etc/shinobisystems/cctv.txt'
fs.stat(etcPath,function(err,stat){
if(err || !stat){
exec('node '+ s.mainDirectory + '/INSTALL/terminalCommands.js',function(err){
if(err)console.log(err)
})
}
next()
return new Promise((resolve, reject) => {
var checkedAdminUsers = {}
console.log('FFmpeg version : '+s.ffmpegVersion)
console.log('Node.js version : '+process.version)
s.processReady = function(){
delete(checkedAdminUsers)
resolve()
s.systemLog(lang.startUpText5)
s.onProcessReadyExtensions.forEach(function(extender){
extender(true)
})
}else{
next()
process.send('ready')
}
}
var loadedAccounts = []
var foundMonitors = []
var loadMonitors = function(callback){
s.beforeMonitorsLoadedOnStartupExtensions.forEach(function(extender){
extender()
})
s.systemLog(lang.startUpText4)
//preliminary monitor start
s.knexQuery({
action: "select",
columns: "*",
table: "Monitors",
},function(err,monitors) {
foundMonitors = monitors
if(err){s.systemLog(err)}
if(monitors && monitors[0]){
var didNotLoad = 0
var loadCompleted = 0
var orphanedVideosForMonitors = {}
var loadMonitor = function(monitor){
const checkAnother = function(){
++loadCompleted
if(monitors[loadCompleted]){
loadMonitor(monitors[loadCompleted])
var checkForTerminalCommands = function(callback){
var next = function(){
if(callback)callback()
}
if(!s.isWin && s.packageJson.mainDirectory !== '.'){
var etcPath = '/etc/shinobisystems/cctv.txt'
fs.stat(etcPath,function(err,stat){
if(err || !stat){
exec('node '+ s.mainDirectory + '/INSTALL/terminalCommands.js',function(err){
if(err)console.log(err)
})
}
next()
})
}else{
next()
}
}
var loadedAccounts = []
var foundMonitors = []
var loadMonitors = function(callback){
s.beforeMonitorsLoadedOnStartupExtensions.forEach(function(extender){
extender()
})
s.systemLog(lang.startUpText4)
//preliminary monitor start
s.knexQuery({
action: "select",
columns: "*",
table: "Monitors",
},function(err,monitors) {
foundMonitors = monitors
if(err){s.systemLog(err)}
if(monitors && monitors[0]){
var didNotLoad = 0
var loadCompleted = 0
var orphanedVideosForMonitors = {}
var loadMonitor = function(monitor){
const checkAnother = function(){
++loadCompleted
if(monitors[loadCompleted]){
loadMonitor(monitors[loadCompleted])
}else{
if(didNotLoad > 0)console.log(`${didNotLoad} Monitor${didNotLoad === 1 ? '' : 's'} not loaded because Admin user does not exist for them. It may have been deleted.`);
callback()
}
}
if(checkedAdminUsers[monitor.ke]){
setTimeout(function(){
if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {}
if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0
s.initiateMonitorObject(monitor)
s.group[monitor.ke].rawMonitorConfigurations[monitor.mid] = monitor
s.sendMonitorStatus({id:monitor.mid,ke:monitor.ke,status:'Stopped'});
var monObj = Object.assign(monitor,{id : monitor.mid})
s.camera(monitor.mode,monObj)
checkAnother()
},1000)
}else{
if(didNotLoad > 0)console.log(`${didNotLoad} Monitor${didNotLoad === 1 ? '' : 's'} not loaded because Admin user does not exist for them. It may have been deleted.`);
callback()
++didNotLoad
checkAnother()
}
}
if(checkedAdminUsers[monitor.ke]){
setTimeout(function(){
if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {}
if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0
s.initiateMonitorObject(monitor)
s.group[monitor.ke].rawMonitorConfigurations[monitor.mid] = monitor
s.sendMonitorStatus({id:monitor.mid,ke:monitor.ke,status:'Stopped'});
var monObj = Object.assign(monitor,{id : monitor.mid})
s.camera(monitor.mode,monObj)
checkAnother()
},1000)
}else{
++didNotLoad
checkAnother()
}
loadMonitor(monitors[loadCompleted])
}else{
callback()
}
loadMonitor(monitors[loadCompleted])
})
}
var checkForOrphanedVideos = function(callback){
var monitors = foundMonitors
if(monitors && monitors[0]){
var loadCompleted = 0
var orphanedVideosForMonitors = {}
var checkForOrphanedVideosForMonitor = function(monitor){
if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {}
if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0
s.orphanedVideoCheck(monitor,null,function(orphanedFilesCount){
if(orphanedFilesCount){
orphanedVideosForMonitors[monitor.ke][monitor.mid] += orphanedFilesCount
}
++loadCompleted
if(monitors[loadCompleted]){
checkForOrphanedVideosForMonitor(monitors[loadCompleted])
}else{
s.systemLog(lang.startUpText6+' : '+s.s(orphanedVideosForMonitors))
delete(foundMonitors)
callback()
}
})
}
checkForOrphanedVideosForMonitor(monitors[loadCompleted])
}else{
callback()
}
})
}
var checkForOrphanedVideos = function(callback){
var monitors = foundMonitors
if(monitors && monitors[0]){
var loadCompleted = 0
var orphanedVideosForMonitors = {}
var checkForOrphanedVideosForMonitor = function(monitor){
if(!orphanedVideosForMonitors[monitor.ke])orphanedVideosForMonitors[monitor.ke] = {}
if(!orphanedVideosForMonitors[monitor.ke][monitor.mid])orphanedVideosForMonitors[monitor.ke][monitor.mid] = 0
s.orphanedVideoCheck(monitor,null,function(orphanedFilesCount){
if(orphanedFilesCount){
orphanedVideosForMonitors[monitor.ke][monitor.mid] += orphanedFilesCount
}
++loadCompleted
if(monitors[loadCompleted]){
checkForOrphanedVideosForMonitor(monitors[loadCompleted])
}else{
s.systemLog(lang.startUpText6+' : '+s.s(orphanedVideosForMonitors))
delete(foundMonitors)
callback()
}
})
}
checkForOrphanedVideosForMonitor(monitors[loadCompleted])
}else{
callback()
}
}
var loadDiskUseForUser = function(user,callback){
s.systemLog(user.mail+' : '+lang.startUpText0)
var userDetails = JSON.parse(user.details)
s.group[user.ke].sizeLimit = parseFloat(userDetails.size) || 10000
s.group[user.ke].sizeLimitVideoPercent = parseFloat(userDetails.size_video_percent) || 90
s.group[user.ke].sizeLimitTimelapseFramesPercent = parseFloat(userDetails.size_timelapse_percent) || 10
s.knexQuery({
action: "select",
columns: "*",
table: "Videos",
where: [
['ke','=',user.ke],
['status','!=',0],
]
},function(err,videos) {
var loadDiskUseForUser = function(user,callback){
s.systemLog(user.mail+' : '+lang.startUpText0)
var userDetails = JSON.parse(user.details)
s.group[user.ke].sizeLimit = parseFloat(userDetails.size) || 10000
s.group[user.ke].sizeLimitVideoPercent = parseFloat(userDetails.size_video_percent) || 90
s.group[user.ke].sizeLimitTimelapseFramesPercent = parseFloat(userDetails.size_timelapse_percent) || 10
s.knexQuery({
action: "select",
columns: "*",
table: "Timelapse Frames",
where: [
['ke','=',user.ke],
]
},function(err,timelapseFrames) {
s.knexQuery({
action: "select",
columns: "*",
table: "Files",
where: [
['ke','=',user.ke],
]
},function(err,files) {
var usedSpaceVideos = 0
var usedSpaceTimelapseFrames = 0
var usedSpaceFilebin = 0
var addStorageData = {
files: [],
videos: [],
timelapeFrames: [],
}
if(videos && videos[0]){
videos.forEach(function(video){
video.details = s.parseJSON(video.details)
if(!video.details.dir){
usedSpaceVideos += video.size
}else{
addStorageData.videos.push(video)
}
})
}
if(timelapseFrames && timelapseFrames[0]){
timelapseFrames.forEach(function(frame){
frame.details = s.parseJSON(frame.details)
if(!frame.details.dir){
usedSpaceTimelapseFrames += frame.size
}else{
addStorageData.timelapeFrames.push(frame)
}
})
}
if(files && files[0]){
files.forEach(function(file){
file.details = s.parseJSON(file.details)
if(!file.details.dir){
usedSpaceFilebin += file.size
}else{
addStorageData.files.push(file)
}
})
}
s.group[user.ke].usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1048576
s.group[user.ke].usedSpaceVideos = usedSpaceVideos / 1048576
s.group[user.ke].usedSpaceFilebin = usedSpaceFilebin / 1048576
s.group[user.ke].usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1048576
loadAddStorageDiskUseForUser(user,addStorageData,function(){
callback()
})
})
})
})
}
var loadCloudDiskUseForUser = function(user,callback){
var userDetails = JSON.parse(user.details)
user.cloudDiskUse = {}
user.size = 0
user.limit = userDetails.size
s.cloudDisksLoaded.forEach(function(storageType){
user.cloudDiskUse[storageType] = {
usedSpace : 0,
firstCount : 0
}
if(s.cloudDiskUseStartupExtensions[storageType])s.cloudDiskUseStartupExtensions[storageType](user,userDetails)
})
var loadCloudVideos = function(callback){
s.knexQuery({
action: "select",
columns: "*",
table: "Cloud Videos",
table: "Videos",
where: [
['ke','=',user.ke],
['status','!=',0],
]
},function(err,videos) {
if(videos && videos[0]){
videos.forEach(function(video){
var storageType = JSON.parse(video.details).type
if(!storageType)storageType = 's3'
var videoSize = video.size / 1048576
user.cloudDiskUse[storageType].usedSpace += videoSize
user.cloudDiskUse[storageType].usedSpaceVideos += videoSize
++user.cloudDiskUse[storageType].firstCount
s.knexQuery({
action: "select",
columns: "*",
table: "Timelapse Frames",
where: [
['ke','=',user.ke],
]
},function(err,timelapseFrames) {
s.knexQuery({
action: "select",
columns: "*",
table: "Files",
where: [
['ke','=',user.ke],
]
},function(err,files) {
var usedSpaceVideos = 0
var usedSpaceTimelapseFrames = 0
var usedSpaceFilebin = 0
var addStorageData = {
files: [],
videos: [],
timelapeFrames: [],
}
if(videos && videos[0]){
videos.forEach(function(video){
video.details = s.parseJSON(video.details)
if(!video.details.dir){
usedSpaceVideos += video.size
}else{
addStorageData.videos.push(video)
}
})
}
if(timelapseFrames && timelapseFrames[0]){
timelapseFrames.forEach(function(frame){
frame.details = s.parseJSON(frame.details)
if(!frame.details.dir){
usedSpaceTimelapseFrames += frame.size
}else{
addStorageData.timelapeFrames.push(frame)
}
})
}
if(files && files[0]){
files.forEach(function(file){
file.details = s.parseJSON(file.details)
if(!file.details.dir){
usedSpaceFilebin += file.size
}else{
addStorageData.files.push(file)
}
})
}
s.group[user.ke].usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1048576
s.group[user.ke].usedSpaceVideos = usedSpaceVideos / 1048576
s.group[user.ke].usedSpaceFilebin = usedSpaceFilebin / 1048576
s.group[user.ke].usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1048576
loadAddStorageDiskUseForUser(user,addStorageData,function(){
callback()
})
})
s.cloudDisksLoaded.forEach(function(storageType){
var firstCount = user.cloudDiskUse[storageType].firstCount
s.systemLog(user.mail+' : '+lang.startUpText1+' : '+firstCount,storageType,user.cloudDiskUse[storageType].usedSpace)
delete(user.cloudDiskUse[storageType].firstCount)
})
}
callback()
})
})
}
var loadCloudTimelapseFrames = function(callback){
var loadCloudDiskUseForUser = function(user,callback){
var userDetails = JSON.parse(user.details)
user.cloudDiskUse = {}
user.size = 0
user.limit = userDetails.size
s.cloudDisksLoaded.forEach(function(storageType){
user.cloudDiskUse[storageType] = {
usedSpace : 0,
firstCount : 0
}
if(s.cloudDiskUseStartupExtensions[storageType])s.cloudDiskUseStartupExtensions[storageType](user,userDetails)
})
var loadCloudVideos = function(callback){
s.knexQuery({
action: "select",
columns: "*",
table: "Cloud Videos",
where: [
['ke','=',user.ke],
['status','!=',0],
]
},function(err,videos) {
if(videos && videos[0]){
videos.forEach(function(video){
var storageType = JSON.parse(video.details).type
if(!storageType)storageType = 's3'
var videoSize = video.size / 1048576
user.cloudDiskUse[storageType].usedSpace += videoSize
user.cloudDiskUse[storageType].usedSpaceVideos += videoSize
++user.cloudDiskUse[storageType].firstCount
})
s.cloudDisksLoaded.forEach(function(storageType){
var firstCount = user.cloudDiskUse[storageType].firstCount
s.systemLog(user.mail+' : '+lang.startUpText1+' : '+firstCount,storageType,user.cloudDiskUse[storageType].usedSpace)
delete(user.cloudDiskUse[storageType].firstCount)
})
}
callback()
})
}
var loadCloudTimelapseFrames = function(callback){
s.knexQuery({
action: "select",
columns: "*",
table: "Cloud Timelapse Frames",
where: [
['ke','=',user.ke],
]
},function(err,frames) {
if(frames && frames[0]){
frames.forEach(function(frame){
var storageType = JSON.parse(frame.details).type
if(!storageType)storageType = 's3'
var frameSize = frame.size / 1048576
user.cloudDiskUse[storageType].usedSpace += frameSize
user.cloudDiskUse[storageType].usedSpaceTimelapseFrames += frameSize
})
}
callback()
})
}
loadCloudVideos(function(){
loadCloudTimelapseFrames(function(){
s.group[user.ke].cloudDiskUse = user.cloudDiskUse
callback()
})
})
}
var loadAddStorageDiskUseForUser = function(user,data,callback){
var videos = data.videos
var timelapseFrames = data.timelapseFrames
var files = data.files
var userDetails = JSON.parse(user.details)
var userAddStorageData = s.parseJSON(userDetails.addStorage) || {}
var currentStorageNumber = 0
var readStorageArray = function(){
var storage = s.listOfStorage[currentStorageNumber]
if(!storage){
//done all checks, move on to next user
callback()
return
}
var path = storage.value
if(path === ''){
++currentStorageNumber
readStorageArray()
return
}
var storageId = path
var storageData = userAddStorageData[storageId] || {}
if(!s.group[user.ke].addStorageUse[storageId])s.group[user.ke].addStorageUse[storageId] = {}
var storageIndex = s.group[user.ke].addStorageUse[storageId]
storageIndex.name = storage.name
storageIndex.path = path
storageIndex.usedSpace = 0
storageIndex.sizeLimit = parseFloat(storageData.limit) || parseFloat(userDetails.size) || 10000
var usedSpaceVideos = 0
var usedSpaceTimelapseFrames = 0
var usedSpaceFilebin = 0
if(videos && videos[0]){
videos.forEach(function(video){
if(video.details.dir === storage.value){
usedSpaceVideos += video.size
}
})
}
if(timelapseFrames && timelapseFrames[0]){
timelapseFrames.forEach(function(frame){
if(video.details.dir === storage.value){
usedSpaceTimelapseFrames += frame.size
}
})
}
if(files && files[0]){
files.forEach(function(file){
if(video.details.dir === storage.value){
usedSpaceFilebin += file.size
}
})
}
storageIndex.usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1048576
storageIndex.usedSpaceVideos = usedSpaceVideos / 1048576
storageIndex.usedSpaceFilebin = usedSpaceFilebin / 1048576
storageIndex.usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1048576
s.systemLog(user.mail+' : '+path+' : '+videos.length,storageIndex.usedSpace)
++currentStorageNumber
readStorageArray()
}
readStorageArray()
}
var loadAdminUsers = function(callback){
//get current disk used for each isolated account (admin user) on startup
s.knexQuery({
action: "select",
columns: "*",
table: "Cloud Timelapse Frames",
table: "Users",
where: [
['ke','=',user.ke],
['details','NOT LIKE','%"sub"%']
]
},function(err,frames) {
if(frames && frames[0]){
frames.forEach(function(frame){
var storageType = JSON.parse(frame.details).type
if(!storageType)storageType = 's3'
var frameSize = frame.size / 1048576
user.cloudDiskUse[storageType].usedSpace += frameSize
user.cloudDiskUse[storageType].usedSpaceTimelapseFrames += frameSize
})
}
callback()
})
}
loadCloudVideos(function(){
loadCloudTimelapseFrames(function(){
s.group[user.ke].cloudDiskUse = user.cloudDiskUse
callback()
})
})
}
var loadAddStorageDiskUseForUser = function(user,data,callback){
var videos = data.videos
var timelapseFrames = data.timelapseFrames
var files = data.files
var userDetails = JSON.parse(user.details)
var userAddStorageData = s.parseJSON(userDetails.addStorage) || {}
var currentStorageNumber = 0
var readStorageArray = function(){
var storage = s.listOfStorage[currentStorageNumber]
if(!storage){
//done all checks, move on to next user
callback()
return
}
var path = storage.value
if(path === ''){
++currentStorageNumber
readStorageArray()
return
}
var storageId = path
var storageData = userAddStorageData[storageId] || {}
if(!s.group[user.ke].addStorageUse[storageId])s.group[user.ke].addStorageUse[storageId] = {}
var storageIndex = s.group[user.ke].addStorageUse[storageId]
storageIndex.name = storage.name
storageIndex.path = path
storageIndex.usedSpace = 0
storageIndex.sizeLimit = parseFloat(storageData.limit) || parseFloat(userDetails.size) || 10000
var usedSpaceVideos = 0
var usedSpaceTimelapseFrames = 0
var usedSpaceFilebin = 0
if(videos && videos[0]){
videos.forEach(function(video){
if(video.details.dir === storage.value){
usedSpaceVideos += video.size
}
})
}
if(timelapseFrames && timelapseFrames[0]){
timelapseFrames.forEach(function(frame){
if(video.details.dir === storage.value){
usedSpaceTimelapseFrames += frame.size
}
})
}
if(files && files[0]){
files.forEach(function(file){
if(video.details.dir === storage.value){
usedSpaceFilebin += file.size
}
})
}
storageIndex.usedSpace = (usedSpaceVideos + usedSpaceTimelapseFrames + usedSpaceFilebin) / 1048576
storageIndex.usedSpaceVideos = usedSpaceVideos / 1048576
storageIndex.usedSpaceFilebin = usedSpaceFilebin / 1048576
storageIndex.usedSpaceTimelapseFrames = usedSpaceTimelapseFrames / 1048576
s.systemLog(user.mail+' : '+path+' : '+videos.length,storageIndex.usedSpace)
++currentStorageNumber
readStorageArray()
}
readStorageArray()
}
var loadAdminUsers = function(callback){
//get current disk used for each isolated account (admin user) on startup
s.knexQuery({
action: "select",
columns: "*",
table: "Users",
where: [
['details','NOT LIKE','%"sub"%']
]
},function(err,users) {
if(users && users[0]){
users.forEach(function(user){
checkedAdminUsers[user.ke] = user
})
var loadLocalDiskUse = function(callback){
var count = users.length
var countFinished = 0
},function(err,users) {
if(users && users[0]){
users.forEach(function(user){
s.loadGroup(user)
s.loadGroupApps(user)
loadedAccounts.push(user.ke)
loadDiskUseForUser(user,function(){
++countFinished
if(countFinished === count){
callback()
}
checkedAdminUsers[user.ke] = user
})
var loadLocalDiskUse = function(callback){
var count = users.length
var countFinished = 0
users.forEach(function(user){
s.loadGroup(user)
s.loadGroupApps(user)
loadedAccounts.push(user.ke)
loadDiskUseForUser(user,function(){
++countFinished
if(countFinished === count){
callback()
}
})
})
}
var loadCloudDiskUse = function(callback){
var count = users.length
var countFinished = 0
users.forEach(function(user){
loadCloudDiskUseForUser(user,function(){
++countFinished
if(countFinished === count){
callback()
}
})
})
}
loadLocalDiskUse(function(){
loadCloudDiskUse(function(){
callback()
})
})
}else{
s.processReady()
}
var loadCloudDiskUse = function(callback){
var count = users.length
var countFinished = 0
users.forEach(function(user){
loadCloudDiskUseForUser(user,function(){
++countFinished
if(countFinished === count){
callback()
}
})
})
}
loadLocalDiskUse(function(){
loadCloudDiskUse(function(){
callback()
})
})
}
config.userHasSubscribed = false
var checkSubscription = function(callback){
var subscriptionFailed = function(){
console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
console.error('This Install of Shinobi is NOT Activated')
console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
s.systemLog('This Install of Shinobi is NOT Activated')
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
console.log('https://licenses.shinobi.video/subscribe')
}
if(config.subscriptionId && config.subscriptionId !== 'sub_XXXXXXXXXXXX'){
var url = 'https://licenses.shinobi.video/subscribe/check?subscriptionId=' + config.subscriptionId
request(url,{
method: 'GET',
timeout: 30000
}, function(err,resp,body){
var json = s.parseJSON(body)
if(err)console.log(err,json)
var hasSubcribed = json && !!json.ok
config.userHasSubscribed = hasSubcribed
callback(hasSubcribed)
if(config.userHasSubscribed){
s.systemLog('This Install of Shinobi is Activated')
if(!json.expired){
s.systemLog(`This License expires on ${json.timeExpires}`)
}
}else{
subscriptionFailed()
}
})
}else{
s.processReady()
subscriptionFailed()
callback(false)
}
})
}
config.userHasSubscribed = false
var checkSubscription = function(callback){
var subscriptionFailed = function(){
console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
console.error('This Install of Shinobi is NOT Activated')
console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
s.systemLog('This Install of Shinobi is NOT Activated')
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
console.log('https://licenses.shinobi.video/subscribe')
}
if(config.subscriptionId && config.subscriptionId !== 'sub_XXXXXXXXXXXX'){
var url = 'https://licenses.shinobi.video/subscribe/check?subscriptionId=' + config.subscriptionId
request(url,{
method: 'GET',
timeout: 30000
}, function(err,resp,body){
var json = s.parseJSON(body)
if(err)console.log(err,json)
var hasSubcribed = json && !!json.ok
config.userHasSubscribed = hasSubcribed
callback(hasSubcribed)
if(config.userHasSubscribed){
s.systemLog('This Install of Shinobi is Activated')
if(!json.expired){
s.systemLog(`This License expires on ${json.timeExpires}`)
}
}else{
subscriptionFailed()
}
})
}else{
subscriptionFailed()
callback(false)
//check disk space every 20 minutes
if(config.autoDropCache===true){
setInterval(function(){
exec('echo 3 > /proc/sys/vm/drop_caches',{detached: true})
},60000*20)
}
}
//check disk space every 20 minutes
if(config.autoDropCache===true){
setInterval(function(){
exec('echo 3 > /proc/sys/vm/drop_caches',{detached: true})
},60000*20)
}
if(config.childNodes.mode !== 'child'){
//master node - startup functions
//hourly check to see if sizePurge has failed to unlock
//checks to see if request count is the number of monitors + 10
s.checkForStalePurgeLocks()
//run prerequsite queries, load users and monitors
//sql/database connection with knex
s.databaseEngine = require('knex')(s.databaseOptions)
//run prerequsite queries
s.preQueries()
setTimeout(() => {
//check for subscription
checkSubscription(function(){
//check terminal commander
checkForTerminalCommands(function(){
//load administrators (groups)
loadAdminUsers(function(){
//load monitors (for groups)
loadMonitors(function(){
//check for orphaned videos
checkForOrphanedVideos(async () => {
s.processReady()
if(config.childNodes.mode !== 'child'){
//master node - startup functions
//hourly check to see if sizePurge has failed to unlock
//checks to see if request count is the number of monitors + 10
s.checkForStalePurgeLocks()
//run prerequsite queries, load users and monitors
//sql/database connection with knex
s.databaseEngine = require('knex')(s.databaseOptions)
//run prerequsite queries
s.preQueries()
setTimeout(() => {
//check for subscription
checkSubscription(function(){
//check terminal commander
checkForTerminalCommands(function(){
//load administrators (groups)
loadAdminUsers(function(){
//load monitors (for groups)
loadMonitors(function(){
//check for orphaned videos
checkForOrphanedVideos(async () => {
s.processReady()
})
})
})
})
})
})
},1500)
}
},1500)
}
})
}

2
libs/webServer.js
View File

@ -57,7 +57,9 @@ module.exports = function(s,config,lang,io){
//SSL options
var wellKnownDirectory = s.mainDirectory + '/web/.well-known'
if(fs.existsSync(wellKnownDirectory))app.use('/.well-known',express.static(wellKnownDirectory))
config.sslEnabled = false
if(config.ssl&&config.ssl.key&&config.ssl.cert){
config.sslEnabled = true
config.ssl.key=fs.readFileSync(s.checkRelativePath(config.ssl.key),'utf8')
config.ssl.cert=fs.readFileSync(s.checkRelativePath(config.ssl.cert),'utf8')
if(config.ssl.port === undefined){

2
web/libs/js/dash2.config.js
View File

@ -3,7 +3,7 @@ $.ccio={
mon:{},
useUTC: <%- config.useUTC || false %>,
definitions: <%-JSON.stringify(define)%>,
libURL: location.search === '?assemble=1' ? location.pathname + '/' : '<%-window.libURL%>',
libURL: location.search === '?p2p=1' ? location.pathname + '/' : '<%-window.libURL%>',
isAppleDevice: navigator.userAgent.match(/(iPod|iPhone|iPad)/)||(navigator.userAgent.match(/(Safari)/)&&!navigator.userAgent.match('Chrome'))
};
<% if(config.DropboxAppKey){ %>

13
web/libs/js/dash2.socketio.js
View File

@ -277,7 +277,7 @@ $.ccio.globalWebsocket=function(d,user){
if($.ccio.op().jpeg_on===true){
$.ccio.init('jpegMode',$.ccio.mon[d.ke+d.id+user.auth_token]);
}else{
if(location.search === '?assemble=1'){
if(location.search === '?p2p=1'){
var path = '/socket.io'
}else{
var path = tool.checkCorrectPathEnding(location.pathname)+'socket.io'
@ -302,7 +302,6 @@ $.ccio.globalWebsocket=function(d,user){
uid: user.uid,
ke: d.ke,
id: d.id,
subscriptionId: subscriptionId,
// channel: channel
})
if(!$.ccio.mon[d.ke+d.id+user.auth_token].ctx||$.ccio.mon[d.ke+d.id+user.auth_token].ctx.length===0){
@ -690,7 +689,7 @@ $.ccio.globalWebsocket=function(d,user){
break;
}
}
if(location.search === '?assemble=1'){
if(location.search === '?p2p=1'){
$user.ws=io(location.origin,{
path : '/socket.io'
});
@ -702,16 +701,16 @@ if(location.search === '?assemble=1'){
$user.ws.on('connect',function (d){
$(document).ready(function(e){
$.ccio.init('id',$user);
if(location.search === '?assemble=1'){
$user.ws.emit('initUser',{
subscriptionId: subscriptionId,
if(location.search === '?p2p=1'){
$user.ws.emit('p2pInitUser',{
user: {
ke: $user.ke,
mail: $user.mail,
auth_token: $user.auth_token,
details: $user.details,
uid: $user.uid,
}
},
machineId: machineId
})
}else{
$.ccio.cx({f:'init',ke:$user.ke,auth:$user.auth_token,uid:$user.uid})

12
web/pages/admin.ejs
View File

@ -86,7 +86,7 @@
</div>
</div>
<script>
var adminApiPrefix = "<%=originalURL%><%=config.webPaths.adminApiPrefix%>"
var adminApiPrefix = location.search === '?p2p=1' ? location.pathname + '/' : "<%=originalURL%><%=config.webPaths.adminApiPrefix%>"
</script>
<% include blocks/confirm.ejs %>
<% include blocks/subpermissions.ejs %>
@ -104,7 +104,7 @@
<script><% include ../libs/js/bootstrap-table.min.js %></script>
<script>
$.ccio={subs:{}};$.ls=localStorage;
if(location.search === '?assemble=1'){
if(location.search === '?p2p=1'){
$.ccio.ws=io(location.origin,{
path : '/socket.io'
});
@ -115,7 +115,7 @@ if(location.search === '?assemble=1'){
}
$.ccio.cx=function(x){if(!x.ke){x.ke=$user.ke;};if(!x.uid){x.uid=$user.uid;};return $.ccio.ws.emit('a',x)}
$.ccio.ws.on('connect',function(d){
$.ccio.cx({f:'init',auth:$user.auth_token});
$.ccio.cx({f:'init',auth:$user.auth_token,machineId: `<%- config.machineId %>`});
})
PNotify.prototype.options.styling = "fontawesome";
$.ccio.ws.on('f',function(d){
@ -179,7 +179,7 @@ $.aN.e.submit(function(e){
e.preventDefault();
e.s = $.aN.e.serializeObject()
e.m = $('#msg').empty()
$.post('<%=originalURL%><%=config.webPaths.adminApiPrefix%>'+$user.auth_token+'/accounts/'+$user.ke+'/register',{data:e.s},function(d){
$.post(adminApiPrefix + $user.auth_token+'/accounts/'+$user.ke+'/register',{data:e.s},function(d){
if(d.msg){
e.m.text(d.msg)
}
@ -197,7 +197,7 @@ $.sU.e.on('click','.delete',function(e){
var html = 'Do you want to delete <b>'+subAccountEmail+'</b>? You cannot recover this account.'
$.confirm.body.html(html)
$.confirm.click({title:'Delete',class:'btn-danger'},function(){
$.post('<%=originalURL%><%=config.webPaths.adminApiPrefix%>'+$user.auth_token+'/accounts/'+$user.ke+'/delete',{
$.post(adminApiPrefix + $user.auth_token+'/accounts/'+$user.ke+'/delete',{
uid: subAccountUid,
mail: subAccountEmail
},function(data){})
@ -269,7 +269,7 @@ $.pR.e.on('change','[monitor]',function(e){
$.pR.f.submit(function(e){
e.preventDefault()
var form = $(this).serializeObject()
$.post('<%=originalURL%><%=config.webPaths.adminApiPrefix%>'+$user.auth_token+'/accounts/'+$user.ke+'/edit',{
$.post(adminApiPrefix + $user.auth_token+'/accounts/'+$user.ke+'/edit',{
uid: $.pR.user,
mail: $.ccio.subs[$.pR.user].mail,
data: form

2
web/pages/blocks/header.ejs
View File

@ -22,4 +22,4 @@
if(!string){string=''}
return string.replace(/'/g,"\\'")
}%>
<script>var subscriptionId = '<%- config.subscriptionId || 'notActivated' %>'</script>
<script>var machineId = '<%- config.machineId || 'noMachineIdSet' %>'</script>

4
web/pages/blocks/mainpermissions.ejs
View File

@ -213,7 +213,7 @@ $.aN.f.submit(function(e){
webPath = 'editAdmin'
postData.account = $.aN.selected
}
$.post('<%=originalURL%><%=config.webPaths.superApiPrefix%>'+$user.sessionKey+'/accounts/'+webPath,postData,function(data){
$.post(superApiPrefix + $user.sessionKey+'/accounts/'+webPath,postData,function(data){
console.log(data)
if(data.ok === true){
$.aN.e.modal('hide')
@ -295,7 +295,7 @@ $.aC.e.on('click','.delete',function(e){
e.html='Do you want to delete <b>'+e.account.mail+'</b>? You cannot recover this account. Files will remain in the filesystem. If you choose to create an account with the same Group Key it will have the previous events activated in that account.'
$.confirm.body.html(e.html)
$.confirm.click({title:'Delete',class:'btn-danger'},function(){
$.post('<%=originalURL%><%=config.webPaths.superApiPrefix%>'+$user.sessionKey+'/accounts/deleteAdmin',{
$.post(superApiPrefix + $user.sessionKey+'/accounts/deleteAdmin',{
account : e.account,
// "deleteSubAccounts" : "1",
// "deleteMonitors" : "1",

6
web/pages/super.ejs
View File

@ -33,7 +33,7 @@
a {cursor:pointer}
</style>
<script>
var superApiPrefix = "<%=originalURL%><%=config.webPaths.superApiPrefix%>"
var superApiPrefix = location.search === '?p2p=1' ? location.pathname + '/' : "<%=originalURL%><%=config.webPaths.superApiPrefix%>"
var shinobiConfig = <%- JSON.stringify(plainConfig) %>
</script>
<% customAutoLoad.superLibsCss.forEach(function(lib){ %>
@ -173,7 +173,7 @@ switch($user.lang){
})
break;
}
if(location.search === '?assemble=1'){
if(location.search === '?p2p=1'){
$.ccio.ws=io(location.origin,{
path : '/socket.io'
});
@ -184,7 +184,7 @@ if(location.search === '?assemble=1'){
}
$.ccio.cx=function(x){return $.ccio.ws.emit('super',x)}
$.ccio.ws.on('connect',function(d){
$.ccio.cx({f:'init',mail:$user.mail,pass:$user.pass})
$.ccio.cx({f:'init',mail:$user.mail,pass:$user.pass,machineId: `<%- config.machineId %>`})
})
$.ccio.ws.on('f',function(d){
switch(d.f){