Merge branch 'plugin-manager-ui' into 'dev'

Plugin Manager UI

See merge request Shinobi-Systems/Shinobi!274
mitchross-coral-installer-update
Moe 2021-01-22 03:15:27 +00:00
commit fda036f8a3
19 changed files with 1302 additions and 75 deletions

View File

@ -56,7 +56,7 @@ require('./libs/ffmpeg.js')(s,config,lang, async () => {
//recording functions
require('./libs/videos.js')(s,config,lang)
//plugins : websocket connected services..
require('./libs/plugins.js')(s,config,lang,io)
require('./libs/plugins.js')(s,config,lang,app,io)
//health : cpu and ram trackers..
require('./libs/health.js')(s,config,lang,io)
//cluster module

View File

@ -507,6 +507,7 @@
"Already exists": "Already exists",
"Creation Interval": "Creation Interval",
"Plugin": "Plugin",
"Plugin Manager": "Plugin Manager",
"IdentityText1": "This is how the system will identify the data for this stream. You cannot change the <b>Monitor ID</b> once you have pressed save. If you want you can make the <b>Monitor ID</b> more human readable before you continue.",
"IdentityText2": "You can duplicate a monitor by modifying the <b>Monitor ID</b> then pressing save. You <b>cannot</b> use the ID of a monitor that already exists or it will save over that monitor's database information.",
"opencvCascadesText": "If you see nothing here then just download this package of <a href=\"https://cdn.shinobi.video/weights/cascades.zip\">cascades</a>. Drop them into <code>plugins/opencv/cascades</code> then press refresh <i class=\"fa fa-retweet\"></i>.",
@ -893,6 +894,7 @@
"No Monitor ID Present in Form": "No Monitor ID Present in Form",
"State Configuration has no monitors associated": "State Configuration has no monitors associated",
"State Configuration Not Found": "State Configuration Not Found",
"Edit Configuration": "Edit Configuration",
"Inserted State Configuration": "Inserted State Configuration",
"Edited State Configuration": "Edited State Configuration",
"Deleted State Configuration": "Deleted State Configuration",

View File

@ -7,9 +7,6 @@ const spawn = require('child_process').spawn
module.exports = async (s,config,lang,app,io) => {
const runningInstallProcesses = {}
const modulesBasePath = s.mainDirectory + '/libs/customAutoLoad/'
const searchText = function(searchFor,searchIn){
return searchIn.indexOf(searchFor) > -1
}
const extractNameFromPackage = (filePath) => {
const filePathParts = filePath.split('/')
const packageName = filePathParts[filePathParts.length - 1].split('.')[0]
@ -222,10 +219,10 @@ module.exports = async (s,config,lang,app,io) => {
var fullPath = thirdLevelName + '/' + filename
var blockPrefix = ''
switch(true){
case searchText('super.',filename):
case filename.contains('super.'):
blockPrefix = 'super'
break;
case searchText('admin.',filename):
case filename.contains('admin.'):
blockPrefix = 'admin'
break;
}

View File

@ -1,8 +1,18 @@
var socketIOclient = require('socket.io-client');
module.exports = function(s,config,lang,io){
module.exports = function(s,config,lang,app,io){
const currentPluginCpuUsage = {}
const currentPluginGpuUsage = {}
const currentPluginFrameProcessingCount = {}
const pluginHandlersSet = {}
const {
triggerEvent,
} = require('./events/utils.js')(s,config,lang)
require('./plugins/superUser.js')(s,config,lang,app,io,{
currentPluginCpuUsage: currentPluginCpuUsage,
currentPluginGpuUsage: currentPluginGpuUsage,
currentPluginFrameProcessingCount: currentPluginFrameProcessingCount,
pluginHandlersSet: pluginHandlersSet,
})
//send data to detector plugin
s.ocvTx = function(data){
// chaining coming in future update
@ -78,10 +88,7 @@ module.exports = function(s,config,lang,io){
s.debugLog(`resetDetectorPluginArray : ${JSON.stringify(pluginArray)}`)
s.detectorPluginArray = pluginArray
}
var currentPluginCpuUsage = {}
var currentPluginGpuUsage = {}
var currentPluginFrameProcessingCount = {}
var pluginHandlersSet = {}
if(config.detectorPluginsCluster){
if(config.clusterUseBasicFrameCount === undefined)config.clusterUseBasicFrameCount = true;
if(config.clusterUseBasicFrameCount){

484
libs/plugins/superUser.js Normal file
View File

@ -0,0 +1,484 @@
const fs = require('fs-extra');
const express = require('express')
const request = require('request')
const unzipper = require('unzipper')
const fetch = require("node-fetch")
const spawn = require('child_process').spawn
const {
Worker
} = require('worker_threads');
module.exports = async (s,config,lang,app,io,currentUse) => {
const {
currentPluginCpuUsage,
currentPluginGpuUsage,
currentPluginFrameProcessingCount,
} = currentUse;
const {
activateClientPlugin,
initializeClientPlugin,
deactivateClientPlugin,
} = require('./utils.js')(s,config,lang)
const {
triggerEvent,
} = require('../events/utils.js')(s,config,lang)
const runningPluginWorkers = {}
const runningInstallProcesses = {}
const modulesBasePath = process.cwd() + '/plugins/'
const extractNameFromPackage = (filePath) => {
const filePathParts = filePath.split('/')
const packageName = filePathParts[filePathParts.length - 1].split('.')[0]
return packageName
}
const getModulePath = (name) => {
return modulesBasePath + name + '/'
}
const getModuleConfiguration = (moduleName) => {
var moduleConfig = {}
const modulePath = modulesBasePath + moduleName
if(fs.existsSync(modulePath + '/conf.json')){
moduleConfig = getModuleProperties(moduleName,'conf')
}else{
if(fs.existsSync(modulePath + '/conf.sample.json')){
moduleConfig = getModuleProperties(moduleName,'conf.sample')
}else{
moduleConfig = {
plug: moduleName.replace('shinobi-',''),
type: 'detector'
}
}
}
return moduleConfig
}
const getModule = (moduleName) => {
const modulePath = modulesBasePath + moduleName
const stats = fs.lstatSync(modulePath)
var newModule;
if(stats.isDirectory()){
newModule = {
name: moduleName,
path: modulePath + '/',
size: stats.size,
lastModified: stats.mtime,
created: stats.ctime,
}
var hasInstaller = false
if(!fs.existsSync(modulePath + '/index.js')){
hasInstaller = true
newModule.noIndex = true
}
//package.json
if(fs.existsSync(modulePath + '/package.json')){
hasInstaller = true
newModule.properties = getModuleProperties(moduleName)
}else{
newModule.properties = {
name: moduleName
}
}
if(newModule.properties.disabled === undefined){
newModule.properties.disabled = true
}
//conf.json
newModule.config = getModuleConfiguration(moduleName)
newModule.hasInstaller = hasInstaller
}
return newModule
}
const getModules = (asArray) => {
const foundModules = {}
fs.readdirSync(modulesBasePath).forEach((moduleName) => {
foundModules[moduleName] = getModule(moduleName)
})
return asArray ? Object.values(foundModules) : foundModules
}
const downloadModule = (downloadUrl,packageName) => {
const downloadPath = modulesBasePath + packageName
fs.mkdirSync(downloadPath)
return new Promise(async (resolve, reject) => {
fs.mkdir(downloadPath, () => {
request(downloadUrl).pipe(fs.createWriteStream(downloadPath + '.zip'))
.on('finish',() => {
zip = fs.createReadStream(downloadPath + '.zip')
.pipe(unzipper.Parse())
.on('entry', async (file) => {
if(file.type === 'Directory'){
try{
fs.mkdirSync(modulesBasePath + file.path, { recursive: true })
}catch(err){
}
}else{
const content = await file.buffer();
fs.writeFile(modulesBasePath + file.path,content,(err) => {
if(err)console.log(err)
})
}
})
.promise()
.then(() => {
fs.remove(downloadPath + '.zip', () => {})
resolve()
})
})
})
})
}
const getModuleProperties = (name,file) => {
const modulePath = getModulePath(name)
const propertiesPath = modulePath + `${file ? file : 'package'}.json`
const properties = fs.existsSync(propertiesPath) ? s.parseJSON(fs.readFileSync(propertiesPath)) : {
name: name
}
return properties
}
const installModule = (name) => {
return new Promise((resolve, reject) => {
if(!runningInstallProcesses[name]){
//depending on module this may only work for Ubuntu
const modulePath = getModulePath(name)
const properties = getModuleProperties(name);
const installerPath = modulePath + `INSTALL.sh`
const propertiesPath = modulePath + 'package.json'
var installProcess
// check for INSTALL.sh (ubuntu only)
if(fs.existsSync(installerPath)){
installProcess = spawn(`sh`,[installerPath])
}else if(fs.existsSync(propertiesPath)){
// no INSTALL.sh found, check for package.json and do `npm install --unsafe-perm`
installProcess = spawn(`npm`,['install','--unsafe-perm','--prefix',modulePath])
}
if(installProcess){
const sendData = (data,channel) => {
const clientData = {
f: 'plugin-info',
module: name,
process: 'install-' + channel,
data: data.toString(),
}
s.tx(clientData,'$')
s.debugLog(clientData)
}
installProcess.stderr.on('data',(data) => {
sendData(data,'stderr')
})
installProcess.stdout.on('data',(data) => {
sendData(data,'stdout')
})
installProcess.on('exit',(data) => {
runningInstallProcesses[name] = null;
})
runningInstallProcesses[name] = installProcess
}
resolve()
}else{
resolve(lang['Already Installing...'])
}
})
}
const disableModule = (name,status) => {
// set status to `false` to enable
const modulePath = getModulePath(name)
const properties = getModuleProperties(name);
const propertiesPath = modulePath + 'package.json'
var packageJson = {
name: name
}
try{
packageJson = JSON.parse(fs.readFileSync(propertiesPath))
}catch(err){
}
packageJson.disabled = status;
fs.writeFileSync(propertiesPath,s.prettyPrint(packageJson))
}
const deleteModule = (name) => {
// requires restart for changes to take effect
try{
const modulePath = modulesBasePath + name
fs.remove(modulePath, (err) => {
console.log(err)
})
return true
}catch(err){
console.log(err)
return false
}
}
const unloadModule = (moduleName) => {
const worker = runningPluginWorkers[moduleName]
if(worker){
worker.terminate()
runningPluginWorkers[moduleName] = null
}
}
const onWorkerMessage = (pluginName,type,data) => {
switch(type){
case'ocv':
switch(data.f){
case'trigger':
triggerEvent(data)
break;
case's.tx':
s.tx(data.data,data.to)
break;
case'log':
s.systemLog('PLUGIN : '+data.plug+' : ',data)
break;
case's.sqlQuery':
s.sqlQuery(data.query,data.values)
break;
case's.knexQuery':
s.knexQuery(data.options)
break;
}
break;
case'cpuUsage':
currentPluginCpuUsage[pluginName] = data
break;
case'gpuUsage':
currentPluginGpuUsage[pluginName] = data
break;
case'processCount':
currentPluginFrameProcessingCount[pluginName] = data
break;
}
}
const loadModule = (shinobiModule) => {
const moduleName = shinobiModule.name
const moduleConfig = shinobiModule.config
const modulePlugName = moduleConfig.plug
const customModulePath = modulesBasePath + '/' + moduleName
const worker = new Worker(customModulePath + '/' + shinobiModule.properties.main,{
workerData: {ok: true}
});
initializeClientPlugin(moduleConfig)
activateClientPlugin(moduleConfig,(data) => {
worker.postMessage(data)
})
worker.on('message', (data) =>{
onWorkerMessage(modulePlugName,...data)
});
worker.on('error', (err) =>{
console.error(err)
});
worker.on('exit', (code) => {
if (code !== 0){
s.debugLog(`Worker (Plugin) stopped with exit code ${code}`);
}
deactivateClientPlugin(modulePlugName)
});
runningPluginWorkers[moduleName] = worker
}
const moveModuleToNameInProperties = (modulePath,packageRoot,properties) => {
return new Promise((resolve,reject) => {
const packageRootParts = packageRoot.split('/')
const filename = packageRootParts[packageRootParts.length - 1]
fs.move(modulePath + packageRoot,modulesBasePath + filename,(err) => {
if(packageRoot){
fs.remove(modulePath, (err) => {
if(err)console.log(err)
resolve(filename)
})
}else{
resolve(filename)
}
})
})
}
const initializeAllModules = async () => {
fs.readdir(modulesBasePath,function(err,folderContents){
if(!err && folderContents.length > 0){
getModules(true).forEach((shinobiModule) => {
if(!shinobiModule || shinobiModule.properties.disabled){
return;
}
loadModule(shinobiModule)
})
}else{
fs.mkdir(modulesBasePath,() => {})
}
})
}
/**
* API : Superuser : Custom Auto Load Package Download.
*/
app.get(config.webPaths.superApiPrefix+':auth/plugins/list', async (req,res) => {
s.superAuth(req.params, async (resp) => {
s.closeJsonResponse(res,{
ok: true,
modules: getModules()
})
},res,req)
})
/**
* API : Superuser : Custom Auto Load Package Download.
*/
app.post(config.webPaths.superApiPrefix+':auth/plugins/download', async (req,res) => {
s.superAuth(req.params, async (resp) => {
try{
const url = req.body.downloadUrl
const packageRoot = req.body.packageRoot || ''
const packageName = req.body.packageName || extractNameFromPackage(url)
const modulePath = getModulePath(packageName)
await downloadModule(url,packageName)
const properties = getModuleProperties(packageName)
const newName = await moveModuleToNameInProperties(modulePath,packageRoot,properties)
const chosenName = newName ? newName : packageName
disableModule(chosenName,true)
s.closeJsonResponse(res,{
ok: true,
moduleName: chosenName,
newModule: getModule(chosenName)
})
}catch(err){
s.closeJsonResponse(res,{
ok: false,
error: err
})
}
},res,req)
})
// /**
// * API : Superuser : Custom Auto Load Package Update.
// */
// app.post(config.webPaths.superApiPrefix+':auth/plugins/update', async (req,res) => {
// s.superAuth(req.params, async (resp) => {
// try{
// const url = req.body.downloadUrl
// const packageRoot = req.body.packageRoot || ''
// const packageName = req.body.packageName || extractNameFromPackage(url)
// const modulePath = getModulePath(packageName)
// await downloadModule(url,packageName)
// const properties = getModuleProperties(packageName)
// const newName = await moveModuleToNameInProperties(modulePath,packageRoot,properties)
// const chosenName = newName ? newName : packageName
//
// disableModule(chosenName,true)
// s.closeJsonResponse(res,{
// ok: true,
// moduleName: chosenName,
// newModule: getModule(chosenName)
// })
// }catch(err){
// s.closeJsonResponse(res,{
// ok: false,
// error: err
// })
// }
// },res,req)
// })
/**
* API : Superuser : Custom Auto Load Package Install.
*/
app.post(config.webPaths.superApiPrefix+':auth/plugins/install', (req,res) => {
s.superAuth(req.params, async (resp) => {
const packageName = req.body.packageName
const cancelInstall = req.body.cancelInstall === 'true' ? true : false
const response = {ok: true}
if(runningInstallProcesses[packageName] && cancelInstall){
runningInstallProcesses[packageName].kill('SIGTERM')
}else{
const error = await installModule(packageName)
if(error){
response.ok = false
response.msg = error
}
}
s.closeJsonResponse(res,response)
},res,req)
})
/**
* API : Superuser : Interact with Installer
*/
app.post(config.webPaths.superApiPrefix+':auth/plugins/command', (req,res) => {
s.superAuth(req.params, async (resp) => {
const packageName = req.body.packageName
const command = req.body.command || ''
const response = {ok: true}
try{
runningInstallProcesses[packageName].stdin.write(`${command}\n`)
}catch(err){
response.ok = false
response.msg = err
}
s.closeJsonResponse(res,response)
},res,req)
})
/**
* API : Superuser : Update Plugin conf.json
*/
app.post(config.webPaths.superApiPrefix+':auth/plugins/configuration/update', (req,res) => {
s.superAuth(req.params, async (resp) => {
const response = {ok: true}
const packageName = req.body.packageName
const configPath = modulesBasePath + packageName + '/conf.json'
const newPluginConfig = s.parseJSON(req.body.config) || {}
try{
await fs.promises.writeFile(configPath,s.prettyPrint(newPluginConfig))
}catch(err){
response.ok = false
response.msg = err
}
s.closeJsonResponse(res,response)
},res,req)
})
/**
* API : Superuser : Get Plugin conf.json
*/
app.get(config.webPaths.superApiPrefix+':auth/plugins/configuration', (req,res) => {
s.superAuth(req.params, async (resp) => {
const response = {ok: true}
const packageName = req.query.packageName
const modulePath = modulesBasePath + packageName
try{
const shinobiModule = getModule(packageName)
response.config = shinobiModule.config
}catch(err){
response.ok = false
response.msg = err
}
s.closeJsonResponse(res,response)
},res,req)
})
/**
* API : Superuser : Custom Auto Load Package set Status (Enabled or Disabled).
*/
app.post(config.webPaths.superApiPrefix+':auth/plugins/status', (req,res) => {
s.superAuth(req.params, async (resp) => {
const status = req.body.status
const packageName = req.body.packageName
const selection = status == 'true' ? true : false
const theModule = getModule(packageName)
disableModule(packageName,selection)
if(theModule.config.hotLoadable === true){
if(!selection){
loadModule(theModule)
}else{
unloadModule(packageName)
}
}
s.closeJsonResponse(res,{ok: true, status: selection})
},res,req)
})
/**
* API : Superuser : Custom Auto Load Package Delete
*/
app.post(config.webPaths.superApiPrefix+':auth/plugins/delete', async (req,res) => {
s.superAuth(req.params, async (resp) => {
const packageName = req.body.packageName
const response = deleteModule(packageName)
s.closeJsonResponse(res,{ok: response})
},res,req)
})
/**
* API : Superuser : Custom Auto Load Package Reload All
*/
app.post(config.webPaths.superApiPrefix+':auth/plugins/reloadAll', async (req,res) => {
s.superAuth(req.params, async (resp) => {
await initializeAllModules();
s.closeJsonResponse(res,{ok: true})
},res,req)
})
// Initialize Modules on Start
await initializeAllModules();
}

49
libs/plugins/utils.js Normal file
View File

@ -0,0 +1,49 @@
module.exports = (s,config,lang) => {
const initializeClientPlugin = (moduleConfig) => {
const modulePlugName = moduleConfig.plug
s.connectedPlugins[modulePlugName] = {
plug: moduleConfig.plug,
type: moduleConfig.type,
plugged: true,
}
}
const activateClientPlugin = (pluginConfig,sendDataToPlugin) => {
s.connectedPlugins[pluginConfig.plug].tx = sendDataToPlugin
//is in client mode (camera.js is client)
// cn.pluginEngine = pluginConfig.plug
s.systemLog('Connected to plugin : Detector - '+pluginConfig.plug+' - '+pluginConfig.type)
switch(pluginConfig.type){
default:
case'detector':
// if(config.oldPluginConnectionMethod)cn.ocv = 1
// cn.detectorPlugin = pluginConfig.plug;
s.addDetectorPlugin(pluginConfig.plug,{
id: s.gid(20),
plug: pluginConfig.plug,
notice: pluginConfig.notice,
isClientPlugin: true,
connectionType: pluginConfig.connectionType
});
s.tx({
f: 'detector_plugged',
plug: pluginConfig.plug,
notice: pluginConfig.notice
},'CPU');
break;
}
}
const deactivateClientPlugin = (modulePlugName) => {
s.connectedPlugins[modulePlugName].plugged = false
s.tx({f:'plugin_engine_unplugged',plug:modulePlugName},'CPU')
s.tx({f:'detector_unplugged',plug:modulePlugName},'CPU')
s.removeDetectorPlugin(modulePlugName)
// s.sendDetectorInfoToClient({f:'detector_plugged'},function(data){
// s.tx(data,'CPU')
// })
}
return {
activateClientPlugin: activateClientPlugin,
initializeClientPlugin: initializeClientPlugin,
deactivateClientPlugin: deactivateClientPlugin,
}
}

View File

@ -12,15 +12,32 @@ var fs = require('fs');
var config = require('./conf.json')
var dotenv = require('dotenv').config()
var s
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
const {
workerData
} = require('worker_threads');
if(workerData && workerData.ok === true){
try{
s = require('./pluginBase.js')(__dirname,config)
s = require('../pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.')
try{
s = require('./pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.')
}
}
}else{
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
try{
s = require('./pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.')
}
}
}
// Base Init />>

View File

@ -16,16 +16,32 @@ var openalpr = {
eu: require ("node-openalpr-shinobi"),
};
var s
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
const {
workerData
} = require('worker_threads');
if(workerData && workerData.ok === true){
try{
s = require('./pluginBase.js')(__dirname,config)
s = require('../pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. This may be because you started this plugin on another machine. Just copy the pluginBase.js file into this (plugin) directory.')
return console.log(config.plug,'pluginBase.js was not found.')
try{
s = require('./pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.')
}
}
}else{
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
try{
s = require('./pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.')
}
}
}
// Base Init />>

306
plugins/pluginWorkerBase.js Normal file
View File

@ -0,0 +1,306 @@
//
// Shinobi - Plugin Base
// Copyright (C) 2016-2025 Moe Alam, moeiscool
//
// # Donate
//
// If you like what I am doing here and want me to continue please consider donating :)
// PayPal : paypal@m03.ca
//
const {
parentPort
} = require('worker_threads');
var fs = require('fs');
var exec = require('child_process').exec;
var spawn = require('child_process').spawn;
var moment = require('moment');
var overAllProcessingCount = 0
module.exports = function(__dirname, config){
if(!config){
return console.log(`Configuration file is missing.`)
}
var plugLog = (d1) => {
console.log(new Date(), config.plug, d1)
}
process.on('uncaughtException', (err) => {
console.error('uncaughtException', err)
})
if(!config.dirname){config.dirname = '.'}
if(!config.port){config.port = 8080}
if(!config.hostPort){config.hostPort = 8082}
if(config.systemLog === undefined){config.systemLog = true}
if(config.connectionType === undefined)config.connectionType = 'websocket'
s = {
group: {},
dir: {},
isWin: (process.platform === 'win32'),
s: (json) => {
return JSON.stringify(json,null,3)
}
}
//default stream folder check
if(!config.streamDir){
if(s.isWin === false){
config.streamDir = '/dev/shm'
}else{
config.streamDir = config.windowsTempDir
}
if(!fs.existsSync(config.streamDir)){
config.streamDir = config.dirname+'/streams/'
}else{
config.streamDir += '/streams/'
}
}
s.dir.streams = config.streamDir
//streams dir
if(!fs.existsSync(s.dir.streams)){
fs.mkdirSync(s.dir.streams)
}
s.checkCorrectPathEnding = (x) => {
var length = x.length
if(x.charAt(length-1) !== '/'){
x = x+'/'
}
return x.replace('__DIR__',config.dirname)
}
s.gid = (x) => {
if(!x){x = 10};
var t = "";
var p = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for( var i = 0; i < x; i++ )
t += p.charAt(Math.floor(Math.random() * p.length));
return t;
};
s.systemLog = (q,w,e) => {
if(!w){w=''}
if(!e){e=''}
if(config.systemLog===true){
return console.log(moment().format(),q,w,e)
}
}
s.debugLog = () => {
if(config.debugLog === true){
console.log(new Date(),arguments)
if(config.debugLogVerbose === true){
console.log(new Error())
}
}
}
s.detectObject = (buffer,d,tx,frameLocation) => {
console.log('detectObject handler not set')
}
const processImage = async (buffer,d,tx,frameLocation) => {
++overAllProcessingCount
sendToParentPort('processCount',overAllProcessingCount)
s.detectObject(buffer,d,tx,frameLocation,() => {
--overAllProcessingCount
sendToParentPort('processCount',overAllProcessingCount)
})
}
const getCpuUsage = (callback) => {
var k = {}
switch(s.platform){
case'win32':
k.cmd = "@for /f \"skip=1\" %p in ('wmic cpu get loadpercentage') do @echo %p%"
break;
case'darwin':
k.cmd = "ps -A -o %cpu | awk '{s+=$1} END {print s}'";
break;
case'linux':
k.cmd = 'top -b -n 2 | awk \'toupper($0) ~ /^.?CPU/ {gsub("id,","100",$8); gsub("%","",$8); print 100-$8}\' | tail -n 1';
break;
case'freebsd':
k.cmd = 'vmstat 1 2 | awk \'END{print 100-$19}\''
break;
case'openbsd':
k.cmd = 'vmstat 1 2 | awk \'END{print 100-$18}\''
break;
}
if(config.customCpuCommand){
exec(config.customCpuCommand,{encoding:'utf8',detached: true},function(err,d){
if(s.isWin===true) {
d = d.replace(/(\r\n|\n|\r)/gm, "").replace(/%/g, "")
}
callback(d)
s.onGetCpuUsageExtensions.forEach(function(extender){
extender(d)
})
})
} else if(k.cmd){
exec(k.cmd,{encoding:'utf8',detached: true},function(err,d){
if(s.isWin===true){
d=d.replace(/(\r\n|\n|\r)/gm,"").replace(/%/g,"")
}
callback(d)
s.onGetCpuUsageExtensions.forEach(function(extender){
extender(d)
})
})
} else {
callback(0)
}
}
const parseNvidiaSmi = function(callback){
var response = {
ok: true,
}
exec('nvidia-smi -x -q',function(err,data){
var response = xmlParser.toJson(data)
var newArray = []
try{
JSON.parse(response).nvidia_smi_log.gpu.forEach((gpu)=>{
newArray.push({
id: gpu.minor_number,
name: gpu.product_name,
brand: gpu.product_brand,
fan_speed: gpu.fan_speed,
temperature: gpu.temperature,
power: gpu.power_readings,
utilization: gpu.utilization,
maxClocks: gpu.max_clocks,
})
})
}catch(err){
}
if(callback)callback(newArray)
})
}
s.onCameraInitExtensions = []
s.onCameraInit = (extender) => {
s.onCameraInitExtensions.push(extender)
}
s.onPluginEvent = []
s.onPluginEventExtender = (extender) => {
s.onPluginEvent.push(extender)
}
s.MainEventController = async (d,cn,tx) => {
switch(d.f){
case'init_plugin_as_host':
if(!cn){
console.log('No CN',d)
return
}
if(d.key!==config.key){
console.log(new Date(),'Plugin Key Mismatch',cn.request.connection.remoteAddress,d)
cn.emit('init',{ok:false})
cn.disconnect()
}else{
console.log(new Date(),'Plugin Connected to Client',cn.request.connection.remoteAddress)
cn.emit('init',{ok:true,plug:config.plug,notice:config.notice,type:config.type})
}
break;
case'init_monitor':
retryConnection = 0
if(s.group[d.ke] && s.group[d.ke][d.id]){
s.group[d.ke][d.id].numberOfTriggers = 0
delete(s.group[d.ke][d.id].cords)
delete(s.group[d.ke][d.id].buffer)
s.onCameraInitExtensions.forEach((extender) => {
extender(d,cn,tx)
})
}
break;
case'frameFromRam':
if(!s.group[d.ke]){
s.group[d.ke] = {}
}
if(!s.group[d.ke][d.id]){
s.group[d.ke][d.id] = {}
}
processImage(buffer,d,tx,d.frameLocation)
break;
case'frame':
try{
if(!s.group[d.ke]){
s.group[d.ke]={}
}
if(!s.group[d.ke][d.id]){
s.group[d.ke][d.id] = {}
s.onCameraInitExtensions.forEach((extender) => {
extender(d,cn,tx)
})
}
if(!s.group[d.ke][d.id].buffer){
s.group[d.ke][d.id].buffer = [d.frame];
}else{
s.group[d.ke][d.id].buffer.push(d.frame)
}
if(d.frame[d.frame.length-2] === 0xFF && d.frame[d.frame.length-1] === 0xD9){
var buffer = Buffer.concat(s.group[d.ke][d.id].buffer);
processImage(buffer,d,tx)
s.group[d.ke][d.id].buffer = null
}
}catch(err){
if(err){
s.systemLog(err)
delete(s.group[d.ke][d.id].buffer)
}
}
break;
}
s.onPluginEvent.forEach((extender) => {
extender(d,cn,tx)
})
}
plugLog('Plugin started as Worker')
const sendToParentPort = (type,data) => {
parentPort.postMessage([type,data])
}
//start plugin as client
parentPort.on('message',(data) => {
s.MainEventController(data,null,s.cx)
})
s.cx = (x) => {
var sendData = Object.assign(x,{
pluginKey : config.key,
plug : config.plug
})
return sendToParentPort('ocv',sendData)
}
if(config.clusterMode){
plugLog('Plugin enabling Cluster Mode...')
if(config.clusterBasedOnGpu){
setTimeout(() => {
parseNvidiaSmi((gpus)=>{
sendToParentPort('gpuUsage',gpus)
})
},1000 * 10)
}else{
setTimeout(() => {
getCpuUsage((percent) => {
sendToParentPort('cpuUsage',percent)
})
},1000 * 10)
}
}
s.getImageDimensions = (d) => {
var height
var width
if(
d.mon.detector_scale_y_object &&
d.mon.detector_scale_y_object !== '' &&
d.mon.detector_scale_x_object &&
d.mon.detector_scale_x_object !== ''
){
height = d.mon.detector_scale_y_object
width = d.mon.detector_scale_x_object
}else{
height = d.mon.detector_scale_y
width = d.mon.detector_scale_x
}
return {
height : parseFloat(height),
width : parseFloat(width)
}
}
return s
}

View File

@ -13,33 +13,50 @@ var fs = require('fs');
var config = require('./conf.json')
var dotenv = require('dotenv').config()
var s
try {
s = require('../pluginBase.js')(__dirname, config)
} catch (err) {
console.log(err)
try {
s = require('./pluginBase.js')(__dirname, config)
} catch (err) {
const {
workerData
} = require('worker_threads');
if(workerData && workerData.ok === true){
try{
s = require('../pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug, 'Plugin start has failed. pluginBase.js was not found.')
try{
s = require('./pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.')
}
}
}else{
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
try{
s = require('./pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.')
}
}
}
var ready = false;
const spawn = require('child_process').spawn;
var child = null
function respawn() {
console.log("respawned python",(new Date()))
const theChild = spawn('python3', ['-u', 'detect_image.py']);
var lastStatusLog = new Date();
theChild.on('exit', () => {
child = respawn();
});
theChild.stdout.on('data', function (data) {
var rawString = data.toString('utf8');
if (new Date() - lastStatusLog > 5000) {
@ -54,7 +71,7 @@ function respawn() {
console.log("Script got error: " + message.data, new Date());
throw message.data;
}
if (obj.type === "info" && obj.data === "ready") {
console.log("set ready true")
ready = true;
@ -67,15 +84,15 @@ function respawn() {
})
return theChild
}
// Base Init />>
child = respawn();
const emptyDataObject = { data: [], type: undefined, time: 0 };
async function process(buffer, type) {
const startTime = new Date();
if (!ready) {
@ -83,7 +100,7 @@ async function process(buffer, type) {
}
ready = false;
child.stdin.write(buffer.toString('base64') + '\n');
var message = null;
await new Promise(resolve => {
child.stdout.once('data', (data) => {
@ -105,8 +122,8 @@ async function process(buffer, type) {
time: new Date() - startTime
}
}
s.detectObject = function (buffer, d, tx, frameLocation, callback) {
process(buffer).then((resp) => {
var results = resp.data

View File

@ -1,7 +1,7 @@
#!/bin/bash
DIR=$(dirname $0)
echo "Removing existing Tensorflow Node.js modules..."
rm -rf node_modules
rm -rf $DIR/node_modules
npm install yarn -g --unsafe-perm --force
wget -O $DIR/package.json https://cdn.shinobi.video/binaries/tensorflow/1.7.3/package.json
GPU_INSTALL="0"

View File

@ -16,7 +16,7 @@ echo $DIR
echo "Replacing package.json for tfjs 2.3.0..."
wget -O $DIR/package.json https://cdn.shinobi.video/binaries/tensorflow/2.3.0/package.json
echo "Removing existing Tensorflow Node.js modules..."
rm -rf node_modules
rm -rf $DIR/node_modules
npm install yarn -g --unsafe-perm --force
installJetsonFlag=false

View File

@ -16,7 +16,7 @@ echo $DIR
echo "Replacing package.json for tfjs 2.3.0..."
wget -O $DIR/package.json https://cdn.shinobi.video/binaries/tensorflow/2.3.0/package.json
echo "Removing existing Tensorflow Node.js modules..."
rm -rf node_modules
rm -rf $DIR/node_modules
npm install yarn -g --unsafe-perm --force
installJetsonFlag=false

View File

@ -2,7 +2,7 @@
DIR=$(dirname $0)
echo "Do not attempt to use this Installer on ARM-based CPUs."
echo "Removing existing Tensorflow Node.js modules..."
rm -rf node_modules
rm -rf $DIR/node_modules
npm install yarn -g --unsafe-perm --force
installJetsonFlag=false

View File

@ -12,28 +12,45 @@ var fs = require('fs');
var config = require('./conf.json')
var dotenv = require('dotenv').config()
var s
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
const {
workerData
} = require('worker_threads');
if(workerData && workerData.ok === true){
try{
s = require('./pluginBase.js')(__dirname,config)
s = require('../pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.')
try{
s = require('./pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.')
}
}
}else{
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
try{
s = require('./pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.')
}
}
const {
haltMessage,
checkStartTime,
setStartTime,
} = require('../pluginCheck.js')
if(!checkStartTime()){
console.log(haltMessage,new Date())
s.disconnectWebSocket()
return
}
setStartTime()
}
const {
haltMessage,
checkStartTime,
setStartTime,
} = require('../pluginCheck.js')
if(!checkStartTime()){
console.log(haltMessage,new Date())
s.disconnectWebSocket()
return
}
setStartTime()
// Base Init />>
const ObjectDetectors = require('./ObjectDetectors.js')(config);

View File

@ -11,15 +11,32 @@
var fs = require('fs');
var config = require('./conf.json')
var s
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
const {
workerData
} = require('worker_threads');
if(workerData && workerData.ok === true){
try{
s = require('./pluginBase.js')(__dirname,config)
s = require('../pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.')
try{
s = require('./pluginWorkerBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.')
}
}
}else{
try{
s = require('../pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
try{
s = require('./pluginBase.js')(__dirname,config)
}catch(err){
console.log(err)
return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.')
}
}
}
// Base Init />>

View File

@ -0,0 +1,278 @@
$(document).ready(function(){
var loadedModules = {}
var listElement = $('#pluginManagerList')
var getModules = function(callback) {
$.get(superApiPrefix + $user.sessionKey + '/plugins/list',callback)
}
var loadedBlocks = {}
var drawModuleBlock = function(module){
var humanName = module.properties.name ? module.properties.name : module.name
if(listElement.find('[package-name="${module.name}"]').length > 0){
var existingElement = listElement.find('[package-name="${module.name}"]')
existingElement.find('.title').text(humanName)
existingElement.find('[plugin-manager-action="status"]').text(module.properties.disabled ? lang.Enable : lang.Disable)
}else{
listElement.append(`
<div class="col-md-12">
<div class="card" package-name="${module.name}">
<div class="card-body">
<div><h4 class="title mt-0">${humanName}</h4></div>
<div><pre><b>${lang['Time Created']} :</b> ${module.created}</pre></div>
<div><pre><b>${lang['Last Modified']} :</b> ${module.lastModified}</pre></div>
<div class="mb-2">
${module.hasInstaller ? `
<a href="#" class="btn btn-sm btn-info" plugin-manager-action="install">${lang['Run Installer']}</a>
<a href="#" class="btn btn-sm btn-danger" style="display:none" plugin-manager-action="cancelInstall">${lang['Stop']}</a>
` : ''}
<a href="#" class="btn btn-sm btn-default" plugin-manager-action="status">${module.properties.disabled ? lang.Enable : lang.Disable}</a>
<a href="#" class="btn btn-sm btn-danger" plugin-manager-action="delete">${lang.Delete}</a>
<a href="#" class="btn btn-sm btn-warning" plugin-manager-action="editConfig">${lang[`Edit Configuration`]}</a>
</div>
<div class="pl-2 pr-2">
<div class="install-output row">
<div class="col-md-6 pr-2"><pre class="install-output-stdout"></pre></div>
<div class="col-md-6 pl-2"><pre class="install-output-stderr"></pre></div>
</div>
<div class="command-installer row" style="display:none">
<div class="col-md-6">
<button type="button" class="btn btn-sm btn-success btn-block" plugin-manager-action="command" command="y">${lang.Yes}</button>
</div>
<div class="col-md-6">
<button type="button" class="btn btn-sm btn-danger btn-block" plugin-manager-action="command" command="N">${lang.No}</button>
</div>
</div>
</div>
</div>
</div>
</div>`)
var newBlock = $(`.card[package-name="${module.name}"]`)
loadedBlocks[module.name] = {
block: newBlock,
stdout: newBlock.find('.install-output-stdout'),
stderr: newBlock.find('.install-output-stderr'),
}
}
}
var downloadModule = function(url,packageRoot,callback){
$.confirm.create({
title: 'Module Download',
body: `Do you want to download the module from ${url}? `,
clickOptions: {
class: 'btn-success',
title: lang.Download,
},
clickCallback: function(){
$.post(superApiPrefix + $user.sessionKey + '/plugins/download',{
downloadUrl: url,
packageRoot: packageRoot,
},callback)
}
})
}
var installModule = function(packageName,callback){
$.confirm.create({
title: 'Install Module',
body: `Do you want to install the module ${packageName}?`,
clickOptions: {
class: 'btn-success',
title: lang.Install,
},
clickCallback: function(){
loadedBlocks[packageName].stdout.empty()
loadedBlocks[packageName].stderr.empty()
$.post(superApiPrefix + $user.sessionKey + '/plugins/install',{
packageName: packageName,
},callback)
}
})
}
var deleteModule = function(packageName,callback){
$.confirm.create({
title: 'Delete Module',
body: `Do you want to delete the module ${packageName}?`,
clickOptions: {
class: 'btn-danger',
title: lang.Delete,
},
clickCallback: function(){
$.post(superApiPrefix + $user.sessionKey + '/plugins/delete',{
packageName: packageName,
},callback)
}
})
}
var setModuleStatus = function(packageName,status,callback){
$.post(superApiPrefix + $user.sessionKey + '/plugins/status',{
status: status,
packageName: packageName,
},callback)
}
var sendInstallerCommand = function(packageName,command,callback){
$.post(superApiPrefix + $user.sessionKey + '/plugins/command',{
command: command,
packageName: packageName,
},callback)
}
var getPluginBlock = function(packageName){
return loadedBlocks[packageName].block
}
var toggleUsabilityOfYesAndNoButtons = function(packageName,enabled){
getPluginBlock(packageName).find('.command-installer')[!enabled ? 'hide' : 'show']()
}
var toggleCardButtons = function(card,buttons){
$.each(buttons,function(n,button){
card.find(`[plugin-manager-action="${button.action}"]`)[button.show ? 'show' : 'hide']()
})
}
$('body').on(`click`,`[plugin-manager-action]`,function(e){
e.preventDefault()
var el = $(this)
var action = el.attr('plugin-manager-action')
var card = el.parents('[package-name]')
var packageName = card.attr('package-name')
switch(action){
case'install':
installModule(packageName,function(data){
if(data.ok){
toggleCardButtons(card,[
{
action: 'install',
show: false,
},
{
action: 'cancelInstall',
show: true,
},
{
action: 'delete',
show: false,
},
{
action: 'status',
show: false,
},
])
}
})
break;
case'cancelInstall':
$.post(superApiPrefix + $user.sessionKey + '/plugins/install',{
packageName: packageName,
cancelInstall: 'true'
},function(data){
if(data.ok){
toggleCardButtons(card,[
{
action: 'install',
show: true,
},
{
action: 'cancelInstall',
show: false,
},
{
action: 'delete',
show: true,
},
{
action: 'status',
show: true,
},
])
}
})
toggleUsabilityOfYesAndNoButtons(packageName,false)
break;
case'status':
setModuleStatus(packageName,!!!loadedModules[packageName].properties.disabled,function(data){
if(data.ok){
loadedModules[packageName].properties.disabled = !!!loadedModules[packageName].properties.disabled
el.text(loadedModules[packageName].properties.disabled ? lang.Enable : lang.Disable)
}
})
break;
case'delete':
deleteModule(packageName,function(data){
if(data.ok){
card.remove()
}
})
break;
case'command':
var command = el.attr('command')
sendInstallerCommand(packageName,command,function(data){
if(data.ok){
toggleUsabilityOfYesAndNoButtons(packageName,false)
}
})
break;
case'editConfig':
$.get(superApiPrefix + $user.sessionKey + '/plugins/configuration?packageName=' + packageName,function(data){
$.confirm.create({
title: lang[`Edit Configuration`],
body: `<textarea id="pluginConfigEditContents" class="form-control" style="height:400px;font-family: monospace;border:1px solid #eee; border-radius: 15px;padding: 10px;">${JSON.stringify(data.config,null,3) || {}}</textarea>`,
clickOptions: {
class: 'btn-success',
title: lang.Save,
},
clickCallback: function(){
var newPluginConfigStringed = $('#pluginConfigEditContents').val()
$.post(superApiPrefix + $user.sessionKey + '/plugins/configuration/update',{
packageName: packageName,
config: newPluginConfigStringed,
},function(data){
console.log(data)
})
}
})
})
break;
}
})
$('#downloadNewModule').submit(function(e){
e.preventDefault();
var el = $(this)
var form = el.serializeObject()
downloadModule(form.downloadUrl,form.packageRoot,function(data){
console.log(data)
if(data.ok){
data.newModule.properties.disabled = true
drawModuleBlock(data.newModule)
}
})
return false
})
setTimeout(function(){
getModules(function(data){
loadedModules = data.modules
console.log(loadedModules)
$.each(data.modules,function(n,module){
drawModuleBlock(module)
})
})
},2000)
$.ccio.ws.on('f',function(data){
switch(data.f){
case'plugin-info':
var name = data.module
switch(data.process){
case'install-stdout':
loadedBlocks[name].stdout.append(`<div class="line">${data.data}</div>`)
// if(loadedBlocks[name].stdout.find('.line').length > 10){
// loadedBlocks[name].stdout.children().first().remove()
// }
if(data.data.indexOf('(y)es or (N)o') > -1){
toggleUsabilityOfYesAndNoButtons(name,true)
}
break;
case'install-stderr':
loadedBlocks[name].stderr.append(`<div class="line">${data.data}</div>`)
// if(loadedBlocks[name].stderr.find('.line').length > 10){
// loadedBlocks[name].stderr.children().first().remove()
// }
break;
}
break;
}
})
})

View File

@ -0,0 +1,14 @@
<link rel="stylesheet" href="<%-window.libURL%>libs/css/super.pluginManager.css">
<form class="form-group-group red pt-4" id="downloadNewPlugin">
<div class="form-group">
<input type="text" placeholder="Download URL for Module" class="form-control" name="downloadUrl" />
</div>
<div class="form-group">
<input type="text" placeholder="Subdirectory for Module (Inside the downloaded package)" class="form-control" name="packageRoot" />
</div>
<div><button type="submit" class="btn btn-block btn-default"><i class="fa fa-download"></i> <%- lang.Download %></button></div>
</form>
<div class="form-group-group red pt-4 pb-0 row m-0" id="pluginManagerList">
</div>
<script src="<%-window.libURL%>libs/js/super.pluginManager.js" type="text/javascript"></script>

View File

@ -94,6 +94,9 @@
<li class="nav-item">
<a class="nav-link" data-toggle="tab" href="#easyRemoteAccess" role="tab"><%-lang['Easy Remote Access (P2P)']%></a>
</li>
<li class="nav-item">
<a class="nav-link" data-toggle="tab" href="#superPluginManager" role="tab"><%-lang['Plugin Manager']%></a>
</li>
</ul>
<div class="card-body">
<!-- Tab panes -->
@ -117,6 +120,9 @@
<div class="tab-pane text-left" id="easyRemoteAccess" role="tabpanel">
<% include blocks/easyRemoteAccess.ejs %>
</div>
<div class="tab-pane text-left" id="superPluginManager" role="tabpanel">
<% include blocks/superPluginManager.ejs %>
</div>
</div>
</div>
</div>