From 69dc2191d784c83db6472be969d49d6e9d4c0b4f Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Tue, 19 Jan 2021 09:04:07 -0800 Subject: [PATCH 1/7] allow enable and disable of plugins as workers --- camera.js | 2 +- languages/en_CA.json | 1 + libs/customAutoLoad.js | 7 +- libs/plugins.js | 3 +- libs/plugins/superUser.js | 383 ++++++++++++++++++++++++ libs/plugins/utils.js | 49 +++ web/libs/js/super.pluginManager.js | 175 +++++++++++ web/pages/blocks/superPluginManager.ejs | 14 + web/pages/super.ejs | 6 + 9 files changed, 633 insertions(+), 7 deletions(-) create mode 100644 libs/plugins/superUser.js create mode 100644 libs/plugins/utils.js create mode 100644 web/libs/js/super.pluginManager.js create mode 100644 web/pages/blocks/superPluginManager.ejs diff --git a/camera.js b/camera.js index 7d492cbd..053fe33d 100644 --- a/camera.js +++ b/camera.js @@ -56,7 +56,7 @@ require('./libs/ffmpeg.js')(s,config,lang, async () => { //recording functions require('./libs/videos.js')(s,config,lang) //plugins : websocket connected services.. - require('./libs/plugins.js')(s,config,lang,io) + require('./libs/plugins.js')(s,config,lang,app,io) //health : cpu and ram trackers.. require('./libs/health.js')(s,config,lang,io) //cluster module diff --git a/languages/en_CA.json b/languages/en_CA.json index 040bf728..d13b4a43 100644 --- a/languages/en_CA.json +++ b/languages/en_CA.json @@ -507,6 +507,7 @@ "Already exists": "Already exists", "Creation Interval": "Creation Interval", "Plugin": "Plugin", + "Plugin Manager": "Plugin Manager", "IdentityText1": "This is how the system will identify the data for this stream. You cannot change the Monitor ID once you have pressed save. If you want you can make the Monitor ID more human readable before you continue.", "IdentityText2": "You can duplicate a monitor by modifying the Monitor ID then pressing save. You cannot use the ID of a monitor that already exists or it will save over that monitor's database information.", "opencvCascadesText": "If you see nothing here then just download this package of cascades. Drop them into plugins/opencv/cascades then press refresh .", diff --git a/libs/customAutoLoad.js b/libs/customAutoLoad.js index af950262..dd725bb5 100644 --- a/libs/customAutoLoad.js +++ b/libs/customAutoLoad.js @@ -7,9 +7,6 @@ const spawn = require('child_process').spawn module.exports = async (s,config,lang,app,io) => { const runningInstallProcesses = {} const modulesBasePath = s.mainDirectory + '/libs/customAutoLoad/' - const searchText = function(searchFor,searchIn){ - return searchIn.indexOf(searchFor) > -1 - } const extractNameFromPackage = (filePath) => { const filePathParts = filePath.split('/') const packageName = filePathParts[filePathParts.length - 1].split('.')[0] @@ -222,10 +219,10 @@ module.exports = async (s,config,lang,app,io) => { var fullPath = thirdLevelName + '/' + filename var blockPrefix = '' switch(true){ - case searchText('super.',filename): + case filename.contains('super.'): blockPrefix = 'super' break; - case searchText('admin.',filename): + case filename.contains('admin.'): blockPrefix = 'admin' break; } diff --git a/libs/plugins.js b/libs/plugins.js index 37ada9d5..3bde7c6d 100644 --- a/libs/plugins.js +++ b/libs/plugins.js @@ -1,8 +1,9 @@ var socketIOclient = require('socket.io-client'); -module.exports = function(s,config,lang,io){ +module.exports = function(s,config,lang,app,io){ const { triggerEvent, } = require('./events/utils.js')(s,config,lang) + require('./plugins/superUser.js')(s,config,lang,app,io) //send data to detector plugin s.ocvTx = function(data){ // chaining coming in future update diff --git a/libs/plugins/superUser.js b/libs/plugins/superUser.js new file mode 100644 index 00000000..4b5406f2 --- /dev/null +++ b/libs/plugins/superUser.js @@ -0,0 +1,383 @@ +const fs = require('fs-extra'); +const express = require('express') +const request = require('request') +const unzipper = require('unzipper') +const fetch = require("node-fetch") +const spawn = require('child_process').spawn +const { + Worker +} = require('worker_threads'); +module.exports = async (s,config,lang,app,io) => { + const { + activateClientPlugin, + initializeClientPlugin, + deactivateClientPlugin, + } = require('./utils.js')(s,config,lang) + const runningPluginWorkers = {} + const runningInstallProcesses = {} + const modulesBasePath = process.cwd() + '/plugins/' + const extractNameFromPackage = (filePath) => { + const filePathParts = filePath.split('/') + const packageName = filePathParts[filePathParts.length - 1].split('.')[0] + return packageName + } + const getModulePath = (name) => { + return modulesBasePath + name + '/' + } + const getModuleConfiguration = (moduleName) => { + var moduleConfig = {} + const modulePath = modulesBasePath + moduleName + if(fs.existsSync(modulePath + '/conf.json')){ + moduleConfig = getModuleProperties(moduleName,'conf') + }else{ + if(fs.existsSync(modulePath + '/conf.sample.json')){ + moduleConfig = getModuleProperties(moduleName,'conf.sample') + }else{ + moduleConfig = { + plug: moduleName.replace('shinobi-',''), + type: 'detector' + } + } + } + return moduleConfig + } + const getModule = (moduleName) => { + const modulePath = modulesBasePath + moduleName + const stats = fs.lstatSync(modulePath) + var newModule; + if(stats.isDirectory()){ + newModule = { + name: moduleName, + path: modulePath + '/', + size: stats.size, + lastModified: stats.mtime, + created: stats.ctime, + } + var hasInstaller = false + if(!fs.existsSync(modulePath + '/index.js')){ + hasInstaller = true + newModule.noIndex = true + } + //package.json + if(fs.existsSync(modulePath + '/package.json')){ + hasInstaller = true + newModule.properties = getModuleProperties(moduleName) + }else{ + newModule.properties = { + name: moduleName + } + } + if(newModule.properties.disabled === undefined){ + newModule.properties.disabled = true + } + //conf.json + newModule.config = getModuleConfiguration(moduleName) + newModule.hasInstaller = hasInstaller + } + return newModule + } + const getModules = (asArray) => { + const foundModules = {} + fs.readdirSync(modulesBasePath).forEach((moduleName) => { + foundModules[moduleName] = getModule(moduleName) + }) + return asArray ? Object.values(foundModules) : foundModules + } + const downloadModule = (downloadUrl,packageName) => { + const downloadPath = modulesBasePath + packageName + fs.mkdirSync(downloadPath) + return new Promise(async (resolve, reject) => { + fs.mkdir(downloadPath, () => { + request(downloadUrl).pipe(fs.createWriteStream(downloadPath + '.zip')) + .on('finish',() => { + zip = fs.createReadStream(downloadPath + '.zip') + .pipe(unzipper.Parse()) + .on('entry', async (file) => { + if(file.type === 'Directory'){ + try{ + fs.mkdirSync(modulesBasePath + file.path, { recursive: true }) + }catch(err){ + + } + }else{ + const content = await file.buffer(); + fs.writeFile(modulesBasePath + file.path,content,(err) => { + if(err)console.log(err) + }) + } + }) + .promise() + .then(() => { + fs.remove(downloadPath + '.zip', () => {}) + resolve() + }) + }) + }) + }) + } + const getModuleProperties = (name,file) => { + const modulePath = getModulePath(name) + const propertiesPath = modulePath + `${file ? file : 'package'}.json` + const properties = fs.existsSync(propertiesPath) ? s.parseJSON(fs.readFileSync(propertiesPath)) : { + name: name + } + return properties + } + const installModule = (name) => { + return new Promise((resolve, reject) => { + if(!runningInstallProcesses[name]){ + //depending on module this may only work for Ubuntu + const modulePath = getModulePath(name) + const properties = getModuleProperties(name); + const installerPath = modulePath + `INSTALL.sh` + const propertiesPath = modulePath + 'package.json' + var installProcess + // check for INSTALL.sh (ubuntu only) + if(fs.existsSync(installerPath)){ + installProcess = spawn(`sh`,[installerPath]) + }else if(fs.existsSync(propertiesPath)){ + // no INSTALL.sh found, check for package.json and do `npm install --unsafe-perm` + installProcess = spawn(`npm`,['install','--unsafe-perm','--prefix',modulePath]) + }else{ + resolve() + } + if(installProcess){ + const sendData = (data,channel) => { + const clientData = { + f: 'module-info', + module: name, + process: 'install-' + channel, + data: data.toString(), + } + s.tx(clientData,'$') + s.debugLog(clientData) + } + installProcess.stderr.on('data',(data) => { + sendData(data,'stderr') + }) + installProcess.stdout.on('data',(data) => { + sendData(data,'stdout') + }) + installProcess.on('exit',(data) => { + runningInstallProcesses[name] = null; + resolve() + }) + runningInstallProcesses[name] = installProcess + } + }else{ + resolve(lang['Already Installing...']) + } + }) + } + const disableModule = (name,status) => { + // set status to `false` to enable + const modulePath = getModulePath(name) + const properties = getModuleProperties(name); + const propertiesPath = modulePath + 'package.json' + var packageJson = { + name: name + } + try{ + packageJson = JSON.parse(fs.readFileSync(propertiesPath)) + }catch(err){ + + } + packageJson.disabled = status; + fs.writeFileSync(propertiesPath,s.prettyPrint(packageJson)) + } + const deleteModule = (name) => { + // requires restart for changes to take effect + try{ + const modulePath = modulesBasePath + name + fs.remove(modulePath, (err) => { + console.log(err) + }) + return true + }catch(err){ + console.log(err) + return false + } + } + const unloadModule = (moduleName) => { + const worker = runningPluginWorkers[moduleName] + if(worker){ + worker.terminate() + runningPluginWorkers[moduleName] = null + } + } + const loadModule = (shinobiModule) => { + console.log(new Error(shinobiModule)) + const moduleName = shinobiModule.name + const moduleConfig = shinobiModule.config + const modulePlugName = moduleConfig.plug + const customModulePath = modulesBasePath + '/' + moduleName + console.log(customModulePath + '/' + shinobiModule.properties.main) + console.log('11111111111111111111111111111111111111') + const worker = new Worker(customModulePath + '/' + shinobiModule.properties.main); + initializeClientPlugin(moduleConfig) + activateClientPlugin(moduleConfig,(data) => { + worker.postMessage(data) + }) + worker.on('error', (err) =>{ + console.error(err) + }); + worker.on('exit', (code) => { + if (code !== 0){ + s.debugLog(`Worker (Plugin) stopped with exit code ${code}`); + } + deactivateClientPlugin(modulePlugName) + }); + runningPluginWorkers[moduleName] = worker + } + const moveModuleToNameInProperties = (modulePath,packageRoot,properties) => { + return new Promise((resolve,reject) => { + const packageRootParts = packageRoot.split('/') + const filename = packageRootParts[packageRootParts.length - 1] + fs.move(modulePath + packageRoot,modulesBasePath + filename,(err) => { + if(packageRoot){ + fs.remove(modulePath, (err) => { + if(err)console.log(err) + resolve(filename) + }) + }else{ + resolve(filename) + } + }) + }) + } + const initializeAllModules = async () => { + fs.readdir(modulesBasePath,function(err,folderContents){ + if(!err && folderContents.length > 0){ + getModules(true).forEach((shinobiModule) => { + if(!shinobiModule || shinobiModule.properties.disabled){ + return; + } + loadModule(shinobiModule) + }) + }else{ + fs.mkdir(modulesBasePath,() => {}) + } + }) + } + /** + * API : Superuser : Custom Auto Load Package Download. + */ + app.get(config.webPaths.superApiPrefix+':auth/plugins/list', async (req,res) => { + s.superAuth(req.params, async (resp) => { + s.closeJsonResponse(res,{ + ok: true, + modules: getModules() + }) + },res,req) + }) + /** + * API : Superuser : Custom Auto Load Package Download. + */ + app.post(config.webPaths.superApiPrefix+':auth/plugins/download', async (req,res) => { + s.superAuth(req.params, async (resp) => { + try{ + const url = req.body.downloadUrl + const packageRoot = req.body.packageRoot || '' + const packageName = req.body.packageName || extractNameFromPackage(url) + const modulePath = getModulePath(packageName) + await downloadModule(url,packageName) + const properties = getModuleProperties(packageName) + const newName = await moveModuleToNameInProperties(modulePath,packageRoot,properties) + const chosenName = newName ? newName : packageName + disableModule(chosenName,true) + s.closeJsonResponse(res,{ + ok: true, + moduleName: chosenName, + newModule: getModule(chosenName) + }) + }catch(err){ + s.closeJsonResponse(res,{ + ok: false, + error: err + }) + } + },res,req) + }) + // /** + // * API : Superuser : Custom Auto Load Package Update. + // */ + // app.post(config.webPaths.superApiPrefix+':auth/plugins/update', async (req,res) => { + // s.superAuth(req.params, async (resp) => { + // try{ + // const url = req.body.downloadUrl + // const packageRoot = req.body.packageRoot || '' + // const packageName = req.body.packageName || extractNameFromPackage(url) + // const modulePath = getModulePath(packageName) + // await downloadModule(url,packageName) + // const properties = getModuleProperties(packageName) + // const newName = await moveModuleToNameInProperties(modulePath,packageRoot,properties) + // const chosenName = newName ? newName : packageName + // + // disableModule(chosenName,true) + // s.closeJsonResponse(res,{ + // ok: true, + // moduleName: chosenName, + // newModule: getModule(chosenName) + // }) + // }catch(err){ + // s.closeJsonResponse(res,{ + // ok: false, + // error: err + // }) + // } + // },res,req) + // }) + /** + * API : Superuser : Custom Auto Load Package Install. + */ + app.post(config.webPaths.superApiPrefix+':auth/plugins/install', (req,res) => { + s.superAuth(req.params, async (resp) => { + const packageName = req.body.packageName + const response = {ok: true} + const error = await installModule(packageName) + if(error){ + response.ok = false + response.msg = error + } + s.closeJsonResponse(res,response) + },res,req) + }) + /** + * API : Superuser : Custom Auto Load Package set Status (Enabled or Disabled). + */ + app.post(config.webPaths.superApiPrefix+':auth/plugins/status', (req,res) => { + s.superAuth(req.params, async (resp) => { + const status = req.body.status + const packageName = req.body.packageName + const selection = status == 'true' ? true : false + disableModule(packageName,selection) + if(!selection){ + loadModule(getModule(packageName)) + }else{ + unloadModule(packageName) + } + s.closeJsonResponse(res,{ok: true, status: selection}) + },res,req) + }) + /** + * API : Superuser : Custom Auto Load Package Delete + */ + app.post(config.webPaths.superApiPrefix+':auth/plugins/delete', async (req,res) => { + s.superAuth(req.params, async (resp) => { + const packageName = req.body.packageName + const response = deleteModule(packageName) + s.closeJsonResponse(res,{ok: response}) + },res,req) + }) + /** + * API : Superuser : Custom Auto Load Package Reload All + */ + app.post(config.webPaths.superApiPrefix+':auth/plugins/reloadAll', async (req,res) => { + s.superAuth(req.params, async (resp) => { + await initializeAllModules(); + s.closeJsonResponse(res,{ok: true}) + },res,req) + }) + // Initialize Modules on Start + await initializeAllModules(); +} diff --git a/libs/plugins/utils.js b/libs/plugins/utils.js new file mode 100644 index 00000000..f7a7927b --- /dev/null +++ b/libs/plugins/utils.js @@ -0,0 +1,49 @@ +module.exports = (s,config,lang) => { + const initializeClientPlugin = (moduleConfig) => { + const modulePlugName = moduleConfig.plug + s.connectedPlugins[modulePlugName] = { + plug: moduleConfig.plug, + type: moduleConfig.type, + plugged: true, + } + } + const activateClientPlugin = (pluginConfig,sendDataToPlugin) => { + s.connectedPlugins[pluginConfig.plug].tx = sendDataToPlugin + //is in client mode (camera.js is client) + // cn.pluginEngine = pluginConfig.plug + s.systemLog('Connected to plugin : Detector - '+pluginConfig.plug+' - '+pluginConfig.type) + switch(pluginConfig.type){ + default: + case'detector': + // if(config.oldPluginConnectionMethod)cn.ocv = 1 + // cn.detectorPlugin = pluginConfig.plug; + s.addDetectorPlugin(pluginConfig.plug,{ + id: s.gid(20), + plug: pluginConfig.plug, + notice: pluginConfig.notice, + isClientPlugin: true, + connectionType: pluginConfig.connectionType + }); + s.tx({ + f: 'detector_plugged', + plug: pluginConfig.plug, + notice: pluginConfig.notice + },'CPU'); + break; + } + } + const deactivateClientPlugin = (modulePlugName) => { + s.connectedPlugins[modulePlugName].plugged = false + s.tx({f:'plugin_engine_unplugged',plug:modulePlugName},'CPU') + s.tx({f:'detector_unplugged',plug:modulePlugName},'CPU') + s.removeDetectorPlugin(modulePlugName) + // s.sendDetectorInfoToClient({f:'detector_plugged'},function(data){ + // s.tx(data,'CPU') + // }) + } + return { + activateClientPlugin: activateClientPlugin, + initializeClientPlugin: initializeClientPlugin, + deactivateClientPlugin: deactivateClientPlugin, + } +} diff --git a/web/libs/js/super.pluginManager.js b/web/libs/js/super.pluginManager.js new file mode 100644 index 00000000..12444164 --- /dev/null +++ b/web/libs/js/super.pluginManager.js @@ -0,0 +1,175 @@ +$(document).ready(function(){ + var loadedModules = {} + var listElement = $('#pluginManagerList') + var getModules = function(callback) { + $.get(superApiPrefix + $user.sessionKey + '/plugins/list',callback) + } + var loadedBlocks = {} + var drawModuleBlock = function(module){ + var humanName = module.properties.name ? module.properties.name : module.name + if(listElement.find('[package-name="${module.name}"]').length > 0){ + var existingElement = listElement.find('[package-name="${module.name}"]') + existingElement.find('.title').text(humanName) + existingElement.find('[plugin-manager-action="status"]').text(module.properties.disabled ? lang.Enable : lang.Disable) + }else{ + listElement.append(` +
+
+
+

${humanName}

+
${lang['Time Created']} : ${module.created}
+
${lang['Last Modified']} : ${module.lastModified}
+
+ ${!module.isIgnitor ? ` + ${module.hasInstaller ? ` + ${lang['Run Installer']} + ` : ''} + ${module.properties.disabled ? lang.Enable : lang.Disable} + ` : ''} + ${lang.Delete} +
+
+
+
+
+
+
+
+
+
`) + var newBlock = $(`.card[package-name="${module.name}"]`) + loadedBlocks[module.name] = { + block: newBlock, + stdout: newBlock.find('.install-output-stdout'), + stderr: newBlock.find('.install-output-stderr'), + } + } + } + var downloadModule = function(url,packageRoot,callback){ + $.confirm.create({ + title: 'Module Download', + body: `Do you want to download the module from ${url}? `, + clickOptions: { + class: 'btn-success', + title: lang.Download, + }, + clickCallback: function(){ + $.post(superApiPrefix + $user.sessionKey + '/plugins/download',{ + downloadUrl: url, + packageRoot: packageRoot, + },callback) + } + }) + } + var installModule = function(packageName,callback){ + $.confirm.create({ + title: 'Install Module', + body: `Do you want to install the module ${packageName}?`, + clickOptions: { + class: 'btn-success', + title: lang.Install, + }, + clickCallback: function(){ + $.post(superApiPrefix + $user.sessionKey + '/plugins/install',{ + packageName: packageName, + },callback) + } + }) + } + var deleteModule = function(packageName,callback){ + $.confirm.create({ + title: 'Delete Module', + body: `Do you want to delete the module ${packageName}?`, + clickOptions: { + class: 'btn-danger', + title: lang.Delete, + }, + clickCallback: function(){ + $.post(superApiPrefix + $user.sessionKey + '/plugins/delete',{ + packageName: packageName, + },callback) + } + }) + } + var setModuleStatus = function(packageName,status,callback){ + $.post(superApiPrefix + $user.sessionKey + '/plugins/status',{ + status: status, + packageName: packageName, + },callback) + } + $('body').on(`click`,`[plugin-manager-action]`,function(e){ + e.preventDefault() + var el = $(this) + var action = el.attr('plugin-manager-action') + var card = el.parents('[package-name]') + console.log(card.length) + var packageName = card.attr('package-name') + switch(action){ + case'install': + installModule(packageName,function(data){ + if(data.ok){ + console.log(data) + } + }) + break; + case'status': + setModuleStatus(packageName,!!!loadedModules[packageName].properties.disabled,function(data){ + if(data.ok){ + loadedModules[packageName].properties.disabled = !!!loadedModules[packageName].properties.disabled + el.text(loadedModules[packageName].properties.disabled ? lang.Enable : lang.Disable) + } + }) + break; + case'delete': + deleteModule(packageName,function(data){ + if(data.ok){ + card.remove() + } + }) + break; + } + }) + $('#downloadNewModule').submit(function(e){ + e.preventDefault(); + var el = $(this) + var form = el.serializeObject() + downloadModule(form.downloadUrl,form.packageRoot,function(data){ + console.log(data) + if(data.ok){ + data.newModule.properties.disabled = true + drawModuleBlock(data.newModule) + } + }) + return false + }) + setTimeout(function(){ + getModules(function(data){ + loadedModules = data.modules + console.log(loadedModules) + $.each(data.modules,function(n,module){ + drawModuleBlock(module) + }) + }) + },2000) + $.ccio.ws.on('f',function(data){ + switch(data.f){ + case'module-info': + var name = data.module + switch(data.process){ + case'install-stdout': + loadedBlocks[name].stdout.append(`
${data.data}
`) + // if(loadedBlocks[name].stdout.find('.line').length > 10){ + // loadedBlocks[name].stdout.children().first().remove() + // } + break; + case'install-stderr': + loadedBlocks[name].stderr.append(`
${data.data}
`) + // if(loadedBlocks[name].stderr.find('.line').length > 10){ + // loadedBlocks[name].stderr.children().first().remove() + // } + break; + } + break; + } + }) +}) diff --git a/web/pages/blocks/superPluginManager.ejs b/web/pages/blocks/superPluginManager.ejs new file mode 100644 index 00000000..1f0d337b --- /dev/null +++ b/web/pages/blocks/superPluginManager.ejs @@ -0,0 +1,14 @@ + +
+
+ +
+
+ +
+
+
+
+ +
+ diff --git a/web/pages/super.ejs b/web/pages/super.ejs index b1679649..a2896f78 100644 --- a/web/pages/super.ejs +++ b/web/pages/super.ejs @@ -94,6 +94,9 @@ +
@@ -117,6 +120,9 @@
<% include blocks/easyRemoteAccess.ejs %>
+
+ <% include blocks/superPluginManager.ejs %> +
From d5b82ec30579fce4fe8107aa9d6e2d1fe55e4d08 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Tue, 19 Jan 2021 09:07:06 -0800 Subject: [PATCH 2/7] disable hotloading plugins from superuser --- libs/plugins/superUser.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/plugins/superUser.js b/libs/plugins/superUser.js index 4b5406f2..81e758c1 100644 --- a/libs/plugins/superUser.js +++ b/libs/plugins/superUser.js @@ -351,11 +351,11 @@ module.exports = async (s,config,lang,app,io) => { const packageName = req.body.packageName const selection = status == 'true' ? true : false disableModule(packageName,selection) - if(!selection){ - loadModule(getModule(packageName)) - }else{ - unloadModule(packageName) - } + // if(!selection){ + // loadModule(getModule(packageName)) + // }else{ + // unloadModule(packageName) + // } s.closeJsonResponse(res,{ok: true, status: selection}) },res,req) }) From 6d56243791b517c61552a8967dc59b938ae7e779 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Tue, 19 Jan 2021 10:15:38 -0800 Subject: [PATCH 3/7] allow hotloading for certain plugins --- libs/plugins/superUser.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/plugins/superUser.js b/libs/plugins/superUser.js index 81e758c1..cfe22f6a 100644 --- a/libs/plugins/superUser.js +++ b/libs/plugins/superUser.js @@ -206,13 +206,10 @@ module.exports = async (s,config,lang,app,io) => { } } const loadModule = (shinobiModule) => { - console.log(new Error(shinobiModule)) const moduleName = shinobiModule.name const moduleConfig = shinobiModule.config const modulePlugName = moduleConfig.plug const customModulePath = modulesBasePath + '/' + moduleName - console.log(customModulePath + '/' + shinobiModule.properties.main) - console.log('11111111111111111111111111111111111111') const worker = new Worker(customModulePath + '/' + shinobiModule.properties.main); initializeClientPlugin(moduleConfig) activateClientPlugin(moduleConfig,(data) => { @@ -350,12 +347,15 @@ module.exports = async (s,config,lang,app,io) => { const status = req.body.status const packageName = req.body.packageName const selection = status == 'true' ? true : false + const theModule = getModule(packageName) disableModule(packageName,selection) - // if(!selection){ - // loadModule(getModule(packageName)) - // }else{ - // unloadModule(packageName) - // } + if(theModule.config.hotLoadable === true){ + if(!selection){ + loadModule() + }else{ + unloadModule(packageName) + } + } s.closeJsonResponse(res,{ok: true, status: selection}) },res,req) }) From ff452e5b2edf6889a6fe62fea550c0b30d1eb140 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Wed, 20 Jan 2021 19:00:45 -0800 Subject: [PATCH 4/7] create pluginWorkerBase to be used with plugins running as Workers --- libs/plugins.js | 16 +- libs/plugins/superUser.js | 51 ++- plugins/face/shinobi-face.js | 29 +- plugins/openalpr/shinobi-openalpr.js | 30 +- plugins/pluginWorkerBase.js | 306 ++++++++++++++++++ .../shinobi-tensorflow-coral.js | 65 ++-- plugins/tensorflow/shinobi-tensorflow.js | 51 ++- plugins/yolo/shinobi-yolo.js | 29 +- 8 files changed, 509 insertions(+), 68 deletions(-) create mode 100644 plugins/pluginWorkerBase.js diff --git a/libs/plugins.js b/libs/plugins.js index 3bde7c6d..ac1f8e68 100644 --- a/libs/plugins.js +++ b/libs/plugins.js @@ -1,9 +1,18 @@ var socketIOclient = require('socket.io-client'); module.exports = function(s,config,lang,app,io){ + const currentPluginCpuUsage = {} + const currentPluginGpuUsage = {} + const currentPluginFrameProcessingCount = {} + const pluginHandlersSet = {} const { triggerEvent, } = require('./events/utils.js')(s,config,lang) - require('./plugins/superUser.js')(s,config,lang,app,io) + require('./plugins/superUser.js')(s,config,lang,app,io,{ + currentPluginCpuUsage: currentPluginCpuUsage, + currentPluginGpuUsage: currentPluginGpuUsage, + currentPluginFrameProcessingCount: currentPluginFrameProcessingCount, + pluginHandlersSet: pluginHandlersSet, + }) //send data to detector plugin s.ocvTx = function(data){ // chaining coming in future update @@ -79,10 +88,7 @@ module.exports = function(s,config,lang,app,io){ s.debugLog(`resetDetectorPluginArray : ${JSON.stringify(pluginArray)}`) s.detectorPluginArray = pluginArray } - var currentPluginCpuUsage = {} - var currentPluginGpuUsage = {} - var currentPluginFrameProcessingCount = {} - var pluginHandlersSet = {} + if(config.detectorPluginsCluster){ if(config.clusterUseBasicFrameCount === undefined)config.clusterUseBasicFrameCount = true; if(config.clusterUseBasicFrameCount){ diff --git a/libs/plugins/superUser.js b/libs/plugins/superUser.js index cfe22f6a..bd37fa66 100644 --- a/libs/plugins/superUser.js +++ b/libs/plugins/superUser.js @@ -7,12 +7,20 @@ const spawn = require('child_process').spawn const { Worker } = require('worker_threads'); -module.exports = async (s,config,lang,app,io) => { +module.exports = async (s,config,lang,app,io,currentUse) => { + const { + currentPluginCpuUsage, + currentPluginGpuUsage, + currentPluginFrameProcessingCount, + } = currentUse; const { activateClientPlugin, initializeClientPlugin, deactivateClientPlugin, } = require('./utils.js')(s,config,lang) + const { + triggerEvent, + } = require('../events/utils.js')(s,config,lang) const runningPluginWorkers = {} const runningInstallProcesses = {} const modulesBasePath = process.cwd() + '/plugins/' @@ -205,16 +213,53 @@ module.exports = async (s,config,lang,app,io) => { runningPluginWorkers[moduleName] = null } } + const onWorkerMessage = (pluginName,type,data) => { + switch(type){ + case'ocv': + switch(data.f){ + case'trigger': + triggerEvent(data) + break; + case's.tx': + s.tx(data.data,data.to) + break; + case'log': + s.systemLog('PLUGIN : '+data.plug+' : ',data) + break; + case's.sqlQuery': + s.sqlQuery(data.query,data.values) + break; + case's.knexQuery': + s.knexQuery(data.options) + break; + } + break; + case'cpuUsage': + currentPluginCpuUsage[pluginName] = data + break; + case'gpuUsage': + currentPluginGpuUsage[pluginName] = data + break; + case'processCount': + currentPluginFrameProcessingCount[pluginName] = data + break; + } + } const loadModule = (shinobiModule) => { const moduleName = shinobiModule.name const moduleConfig = shinobiModule.config const modulePlugName = moduleConfig.plug const customModulePath = modulesBasePath + '/' + moduleName - const worker = new Worker(customModulePath + '/' + shinobiModule.properties.main); + const worker = new Worker(customModulePath + '/' + shinobiModule.properties.main,{ + workerData: {ok: true} + }); initializeClientPlugin(moduleConfig) activateClientPlugin(moduleConfig,(data) => { worker.postMessage(data) }) + worker.on('message', (data) =>{ + onWorkerMessage(modulePlugName,...data) + }); worker.on('error', (err) =>{ console.error(err) }); @@ -351,7 +396,7 @@ module.exports = async (s,config,lang,app,io) => { disableModule(packageName,selection) if(theModule.config.hotLoadable === true){ if(!selection){ - loadModule() + loadModule(theModule) }else{ unloadModule(packageName) } diff --git a/plugins/face/shinobi-face.js b/plugins/face/shinobi-face.js index 5f6731ee..5bff1469 100644 --- a/plugins/face/shinobi-face.js +++ b/plugins/face/shinobi-face.js @@ -12,15 +12,32 @@ var fs = require('fs'); var config = require('./conf.json') var dotenv = require('dotenv').config() var s -try{ - s = require('../pluginBase.js')(__dirname,config) -}catch(err){ - console.log(err) +const { + workerData +} = require('worker_threads'); +if(workerData && workerData.ok === true){ try{ - s = require('./pluginBase.js')(__dirname,config) + s = require('../pluginWorkerBase.js')(__dirname,config) }catch(err){ console.log(err) - return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + try{ + s = require('./pluginWorkerBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.') + } + } +}else{ + try{ + s = require('../pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + try{ + s = require('./pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + } } } // Base Init />> diff --git a/plugins/openalpr/shinobi-openalpr.js b/plugins/openalpr/shinobi-openalpr.js index fcb34cc2..7c42ef8f 100644 --- a/plugins/openalpr/shinobi-openalpr.js +++ b/plugins/openalpr/shinobi-openalpr.js @@ -16,16 +16,32 @@ var openalpr = { eu: require ("node-openalpr-shinobi"), }; var s -try{ - s = require('../pluginBase.js')(__dirname,config) -}catch(err){ - console.log(err) +const { + workerData +} = require('worker_threads'); +if(workerData && workerData.ok === true){ try{ - s = require('./pluginBase.js')(__dirname,config) + s = require('../pluginWorkerBase.js')(__dirname,config) }catch(err){ console.log(err) - return console.log(config.plug,'Plugin start has failed. This may be because you started this plugin on another machine. Just copy the pluginBase.js file into this (plugin) directory.') - return console.log(config.plug,'pluginBase.js was not found.') + try{ + s = require('./pluginWorkerBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.') + } + } +}else{ + try{ + s = require('../pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + try{ + s = require('./pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + } } } // Base Init />> diff --git a/plugins/pluginWorkerBase.js b/plugins/pluginWorkerBase.js new file mode 100644 index 00000000..69ae375a --- /dev/null +++ b/plugins/pluginWorkerBase.js @@ -0,0 +1,306 @@ +// +// Shinobi - Plugin Base +// Copyright (C) 2016-2025 Moe Alam, moeiscool +// +// # Donate +// +// If you like what I am doing here and want me to continue please consider donating :) +// PayPal : paypal@m03.ca +// +const { + parentPort +} = require('worker_threads'); +var fs = require('fs'); +var exec = require('child_process').exec; +var spawn = require('child_process').spawn; +var moment = require('moment'); +var overAllProcessingCount = 0 +module.exports = function(__dirname, config){ + if(!config){ + return console.log(`Configuration file is missing.`) + } + var plugLog = (d1) => { + console.log(new Date(), config.plug, d1) + } + + process.on('uncaughtException', (err) => { + console.error('uncaughtException', err) + }) + + if(!config.dirname){config.dirname = '.'} + if(!config.port){config.port = 8080} + if(!config.hostPort){config.hostPort = 8082} + if(config.systemLog === undefined){config.systemLog = true} + if(config.connectionType === undefined)config.connectionType = 'websocket' + s = { + group: {}, + dir: {}, + isWin: (process.platform === 'win32'), + s: (json) => { + return JSON.stringify(json,null,3) + } + } + //default stream folder check + if(!config.streamDir){ + if(s.isWin === false){ + config.streamDir = '/dev/shm' + }else{ + config.streamDir = config.windowsTempDir + } + if(!fs.existsSync(config.streamDir)){ + config.streamDir = config.dirname+'/streams/' + }else{ + config.streamDir += '/streams/' + } + } + s.dir.streams = config.streamDir + //streams dir + if(!fs.existsSync(s.dir.streams)){ + fs.mkdirSync(s.dir.streams) + } + s.checkCorrectPathEnding = (x) => { + var length = x.length + if(x.charAt(length-1) !== '/'){ + x = x+'/' + } + return x.replace('__DIR__',config.dirname) + } + s.gid = (x) => { + if(!x){x = 10}; + var t = ""; + var p = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + for( var i = 0; i < x; i++ ) + t += p.charAt(Math.floor(Math.random() * p.length)); + return t; + }; + s.systemLog = (q,w,e) => { + if(!w){w=''} + if(!e){e=''} + if(config.systemLog===true){ + return console.log(moment().format(),q,w,e) + } + } + s.debugLog = () => { + if(config.debugLog === true){ + console.log(new Date(),arguments) + if(config.debugLogVerbose === true){ + console.log(new Error()) + } + } + } + s.detectObject = (buffer,d,tx,frameLocation) => { + console.log('detectObject handler not set') + } + const processImage = async (buffer,d,tx,frameLocation) => { + ++overAllProcessingCount + sendToParentPort('processCount',overAllProcessingCount) + s.detectObject(buffer,d,tx,frameLocation,() => { + --overAllProcessingCount + sendToParentPort('processCount',overAllProcessingCount) + }) + } + const getCpuUsage = (callback) => { + var k = {} + switch(s.platform){ + case'win32': + k.cmd = "@for /f \"skip=1\" %p in ('wmic cpu get loadpercentage') do @echo %p%" + break; + case'darwin': + k.cmd = "ps -A -o %cpu | awk '{s+=$1} END {print s}'"; + break; + case'linux': + k.cmd = 'top -b -n 2 | awk \'toupper($0) ~ /^.?CPU/ {gsub("id,","100",$8); gsub("%","",$8); print 100-$8}\' | tail -n 1'; + break; + case'freebsd': + k.cmd = 'vmstat 1 2 | awk \'END{print 100-$19}\'' + break; + case'openbsd': + k.cmd = 'vmstat 1 2 | awk \'END{print 100-$18}\'' + break; + } + if(config.customCpuCommand){ + exec(config.customCpuCommand,{encoding:'utf8',detached: true},function(err,d){ + if(s.isWin===true) { + d = d.replace(/(\r\n|\n|\r)/gm, "").replace(/%/g, "") + } + callback(d) + s.onGetCpuUsageExtensions.forEach(function(extender){ + extender(d) + }) + }) + } else if(k.cmd){ + exec(k.cmd,{encoding:'utf8',detached: true},function(err,d){ + if(s.isWin===true){ + d=d.replace(/(\r\n|\n|\r)/gm,"").replace(/%/g,"") + } + callback(d) + s.onGetCpuUsageExtensions.forEach(function(extender){ + extender(d) + }) + }) + } else { + callback(0) + } + } + const parseNvidiaSmi = function(callback){ + var response = { + ok: true, + } + exec('nvidia-smi -x -q',function(err,data){ + var response = xmlParser.toJson(data) + var newArray = [] + try{ + JSON.parse(response).nvidia_smi_log.gpu.forEach((gpu)=>{ + newArray.push({ + id: gpu.minor_number, + name: gpu.product_name, + brand: gpu.product_brand, + fan_speed: gpu.fan_speed, + temperature: gpu.temperature, + power: gpu.power_readings, + utilization: gpu.utilization, + maxClocks: gpu.max_clocks, + }) + }) + }catch(err){ + + } + if(callback)callback(newArray) + }) + } + s.onCameraInitExtensions = [] + s.onCameraInit = (extender) => { + s.onCameraInitExtensions.push(extender) + } + s.onPluginEvent = [] + s.onPluginEventExtender = (extender) => { + s.onPluginEvent.push(extender) + } + s.MainEventController = async (d,cn,tx) => { + switch(d.f){ + case'init_plugin_as_host': + if(!cn){ + console.log('No CN',d) + return + } + if(d.key!==config.key){ + console.log(new Date(),'Plugin Key Mismatch',cn.request.connection.remoteAddress,d) + cn.emit('init',{ok:false}) + cn.disconnect() + }else{ + console.log(new Date(),'Plugin Connected to Client',cn.request.connection.remoteAddress) + cn.emit('init',{ok:true,plug:config.plug,notice:config.notice,type:config.type}) + } + break; + case'init_monitor': + retryConnection = 0 + if(s.group[d.ke] && s.group[d.ke][d.id]){ + s.group[d.ke][d.id].numberOfTriggers = 0 + delete(s.group[d.ke][d.id].cords) + delete(s.group[d.ke][d.id].buffer) + s.onCameraInitExtensions.forEach((extender) => { + extender(d,cn,tx) + }) + } + break; + case'frameFromRam': + if(!s.group[d.ke]){ + s.group[d.ke] = {} + } + if(!s.group[d.ke][d.id]){ + s.group[d.ke][d.id] = {} + } + processImage(buffer,d,tx,d.frameLocation) + break; + case'frame': + try{ + if(!s.group[d.ke]){ + s.group[d.ke]={} + } + if(!s.group[d.ke][d.id]){ + s.group[d.ke][d.id] = {} + s.onCameraInitExtensions.forEach((extender) => { + extender(d,cn,tx) + }) + } + if(!s.group[d.ke][d.id].buffer){ + s.group[d.ke][d.id].buffer = [d.frame]; + }else{ + s.group[d.ke][d.id].buffer.push(d.frame) + } + if(d.frame[d.frame.length-2] === 0xFF && d.frame[d.frame.length-1] === 0xD9){ + var buffer = Buffer.concat(s.group[d.ke][d.id].buffer); + processImage(buffer,d,tx) + s.group[d.ke][d.id].buffer = null + } + }catch(err){ + if(err){ + s.systemLog(err) + delete(s.group[d.ke][d.id].buffer) + } + } + break; + } + s.onPluginEvent.forEach((extender) => { + extender(d,cn,tx) + }) + } + plugLog('Plugin started as Worker') + const sendToParentPort = (type,data) => { + parentPort.postMessage([type,data]) + } + + //start plugin as client + + parentPort.on('message',(data) => { + s.MainEventController(data,null,s.cx) + }) + + + s.cx = (x) => { + var sendData = Object.assign(x,{ + pluginKey : config.key, + plug : config.plug + }) + return sendToParentPort('ocv',sendData) + } + + if(config.clusterMode){ + plugLog('Plugin enabling Cluster Mode...') + if(config.clusterBasedOnGpu){ + setTimeout(() => { + parseNvidiaSmi((gpus)=>{ + sendToParentPort('gpuUsage',gpus) + }) + },1000 * 10) + }else{ + setTimeout(() => { + getCpuUsage((percent) => { + sendToParentPort('cpuUsage',percent) + }) + },1000 * 10) + } + } + + s.getImageDimensions = (d) => { + var height + var width + if( + d.mon.detector_scale_y_object && + d.mon.detector_scale_y_object !== '' && + d.mon.detector_scale_x_object && + d.mon.detector_scale_x_object !== '' + ){ + height = d.mon.detector_scale_y_object + width = d.mon.detector_scale_x_object + }else{ + height = d.mon.detector_scale_y + width = d.mon.detector_scale_x + } + return { + height : parseFloat(height), + width : parseFloat(width) + } + } + return s +} diff --git a/plugins/tensorflow-coral/shinobi-tensorflow-coral.js b/plugins/tensorflow-coral/shinobi-tensorflow-coral.js index f7d5d8b9..97ab2e93 100644 --- a/plugins/tensorflow-coral/shinobi-tensorflow-coral.js +++ b/plugins/tensorflow-coral/shinobi-tensorflow-coral.js @@ -13,33 +13,50 @@ var fs = require('fs'); var config = require('./conf.json') var dotenv = require('dotenv').config() var s -try { - s = require('../pluginBase.js')(__dirname, config) -} catch (err) { - console.log(err) - try { - s = require('./pluginBase.js')(__dirname, config) - } catch (err) { +const { + workerData +} = require('worker_threads'); +if(workerData && workerData.ok === true){ + try{ + s = require('../pluginWorkerBase.js')(__dirname,config) + }catch(err){ console.log(err) - return console.log(config.plug, 'Plugin start has failed. pluginBase.js was not found.') + try{ + s = require('./pluginWorkerBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.') + } + } +}else{ + try{ + s = require('../pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + try{ + s = require('./pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + } } } - - + + var ready = false; const spawn = require('child_process').spawn; var child = null function respawn() { - + console.log("respawned python",(new Date())) const theChild = spawn('python3', ['-u', 'detect_image.py']); - + var lastStatusLog = new Date(); - + theChild.on('exit', () => { child = respawn(); }); - + theChild.stdout.on('data', function (data) { var rawString = data.toString('utf8'); if (new Date() - lastStatusLog > 5000) { @@ -54,7 +71,7 @@ function respawn() { console.log("Script got error: " + message.data, new Date()); throw message.data; } - + if (obj.type === "info" && obj.data === "ready") { console.log("set ready true") ready = true; @@ -67,15 +84,15 @@ function respawn() { }) return theChild } - - - - + + + + // Base Init />> child = respawn(); - + const emptyDataObject = { data: [], type: undefined, time: 0 }; - + async function process(buffer, type) { const startTime = new Date(); if (!ready) { @@ -83,7 +100,7 @@ async function process(buffer, type) { } ready = false; child.stdin.write(buffer.toString('base64') + '\n'); - + var message = null; await new Promise(resolve => { child.stdout.once('data', (data) => { @@ -105,8 +122,8 @@ async function process(buffer, type) { time: new Date() - startTime } } - - + + s.detectObject = function (buffer, d, tx, frameLocation, callback) { process(buffer).then((resp) => { var results = resp.data diff --git a/plugins/tensorflow/shinobi-tensorflow.js b/plugins/tensorflow/shinobi-tensorflow.js index a52770b9..2aaf2041 100644 --- a/plugins/tensorflow/shinobi-tensorflow.js +++ b/plugins/tensorflow/shinobi-tensorflow.js @@ -12,28 +12,45 @@ var fs = require('fs'); var config = require('./conf.json') var dotenv = require('dotenv').config() var s -try{ - s = require('../pluginBase.js')(__dirname,config) -}catch(err){ - console.log(err) +const { + workerData +} = require('worker_threads'); +if(workerData && workerData.ok === true){ try{ - s = require('./pluginBase.js')(__dirname,config) + s = require('../pluginWorkerBase.js')(__dirname,config) }catch(err){ console.log(err) - return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + try{ + s = require('./pluginWorkerBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.') + } } +}else{ + try{ + s = require('../pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + try{ + s = require('./pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + } + } + const { + haltMessage, + checkStartTime, + setStartTime, + } = require('../pluginCheck.js') + if(!checkStartTime()){ + console.log(haltMessage,new Date()) + s.disconnectWebSocket() + return + } + setStartTime() } -const { - haltMessage, - checkStartTime, - setStartTime, -} = require('../pluginCheck.js') -if(!checkStartTime()){ - console.log(haltMessage,new Date()) - s.disconnectWebSocket() - return -} -setStartTime() // Base Init />> const ObjectDetectors = require('./ObjectDetectors.js')(config); diff --git a/plugins/yolo/shinobi-yolo.js b/plugins/yolo/shinobi-yolo.js index 1c518127..b2e1e845 100644 --- a/plugins/yolo/shinobi-yolo.js +++ b/plugins/yolo/shinobi-yolo.js @@ -11,15 +11,32 @@ var fs = require('fs'); var config = require('./conf.json') var s -try{ - s = require('../pluginBase.js')(__dirname,config) -}catch(err){ - console.log(err) +const { + workerData +} = require('worker_threads'); +if(workerData && workerData.ok === true){ try{ - s = require('./pluginBase.js')(__dirname,config) + s = require('../pluginWorkerBase.js')(__dirname,config) }catch(err){ console.log(err) - return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + try{ + s = require('./pluginWorkerBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'WORKER : Plugin start has failed. pluginBase.js was not found.') + } + } +}else{ + try{ + s = require('../pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + try{ + s = require('./pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + } } } // Base Init />> From 88d5bb347936a061775bfb9e282fbec1a15f6718 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Thu, 21 Jan 2021 05:50:46 -0800 Subject: [PATCH 5/7] Plugin Manager : allow interacting with Installer from UI --- libs/plugins/superUser.js | 19 +++++++++++++++- plugins/tensorflow/INSTALL-1-7-3.sh | 2 +- plugins/tensorflow/INSTALL-2-3-0.sh | 2 +- plugins/tensorflow/INSTALL-jetson.sh | 2 +- plugins/tensorflow/INSTALL.sh | 2 +- web/libs/js/super.pluginManager.js | 33 +++++++++++++++++++++++++++- 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/libs/plugins/superUser.js b/libs/plugins/superUser.js index bd37fa66..e4fae68e 100644 --- a/libs/plugins/superUser.js +++ b/libs/plugins/superUser.js @@ -152,7 +152,7 @@ module.exports = async (s,config,lang,app,io,currentUse) => { if(installProcess){ const sendData = (data,channel) => { const clientData = { - f: 'module-info', + f: 'plugin-info', module: name, process: 'install-' + channel, data: data.toString(), @@ -385,6 +385,23 @@ module.exports = async (s,config,lang,app,io,currentUse) => { },res,req) }) /** + * API : Superuser : Interact with Installer + */ + app.post(config.webPaths.superApiPrefix+':auth/plugins/command', (req,res) => { + s.superAuth(req.params, async (resp) => { + const packageName = req.body.packageName + const command = req.body.command || '' + const response = {ok: true} + try{ + runningInstallProcesses[packageName].stdin.write(`${command}\n`) + }catch(err){ + response.ok = false + response.msg = error + } + s.closeJsonResponse(res,response) + },res,req) + }) + /** * API : Superuser : Custom Auto Load Package set Status (Enabled or Disabled). */ app.post(config.webPaths.superApiPrefix+':auth/plugins/status', (req,res) => { diff --git a/plugins/tensorflow/INSTALL-1-7-3.sh b/plugins/tensorflow/INSTALL-1-7-3.sh index 22222610..f0edf0c3 100644 --- a/plugins/tensorflow/INSTALL-1-7-3.sh +++ b/plugins/tensorflow/INSTALL-1-7-3.sh @@ -1,7 +1,7 @@ #!/bin/bash DIR=$(dirname $0) echo "Removing existing Tensorflow Node.js modules..." -rm -rf node_modules +rm -rf $DIR/node_modules npm install yarn -g --unsafe-perm --force wget -O $DIR/package.json https://cdn.shinobi.video/binaries/tensorflow/1.7.3/package.json GPU_INSTALL="0" diff --git a/plugins/tensorflow/INSTALL-2-3-0.sh b/plugins/tensorflow/INSTALL-2-3-0.sh index c0107eb3..bc24dc12 100644 --- a/plugins/tensorflow/INSTALL-2-3-0.sh +++ b/plugins/tensorflow/INSTALL-2-3-0.sh @@ -16,7 +16,7 @@ echo $DIR echo "Replacing package.json for tfjs 2.3.0..." wget -O $DIR/package.json https://cdn.shinobi.video/binaries/tensorflow/2.3.0/package.json echo "Removing existing Tensorflow Node.js modules..." -rm -rf node_modules +rm -rf $DIR/node_modules npm install yarn -g --unsafe-perm --force installJetsonFlag=false diff --git a/plugins/tensorflow/INSTALL-jetson.sh b/plugins/tensorflow/INSTALL-jetson.sh index c0107eb3..bc24dc12 100644 --- a/plugins/tensorflow/INSTALL-jetson.sh +++ b/plugins/tensorflow/INSTALL-jetson.sh @@ -16,7 +16,7 @@ echo $DIR echo "Replacing package.json for tfjs 2.3.0..." wget -O $DIR/package.json https://cdn.shinobi.video/binaries/tensorflow/2.3.0/package.json echo "Removing existing Tensorflow Node.js modules..." -rm -rf node_modules +rm -rf $DIR/node_modules npm install yarn -g --unsafe-perm --force installJetsonFlag=false diff --git a/plugins/tensorflow/INSTALL.sh b/plugins/tensorflow/INSTALL.sh index 4533da40..cc90520b 100644 --- a/plugins/tensorflow/INSTALL.sh +++ b/plugins/tensorflow/INSTALL.sh @@ -2,7 +2,7 @@ DIR=$(dirname $0) echo "Do not attempt to use this Installer on ARM-based CPUs." echo "Removing existing Tensorflow Node.js modules..." -rm -rf node_modules +rm -rf $DIR/node_modules npm install yarn -g --unsafe-perm --force installJetsonFlag=false diff --git a/web/libs/js/super.pluginManager.js b/web/libs/js/super.pluginManager.js index 12444164..13ba571a 100644 --- a/web/libs/js/super.pluginManager.js +++ b/web/libs/js/super.pluginManager.js @@ -33,6 +33,14 @@ $(document).ready(function(){
+ @@ -97,6 +105,18 @@ $(document).ready(function(){ packageName: packageName, },callback) } + var sendInstallerCommand = function(packageName,command,callback){ + $.post(superApiPrefix + $user.sessionKey + '/plugins/command',{ + command: command, + packageName: packageName, + },callback) + } + var getPluginBlock = (packageName) => { + return listElement.find(`[package-name="${packageName}"]`) + } + var toggleUsabilityOfYesAndNoButtons = function(packageName,enabled){ + getPluginBlock(packageName).find('.command-installer')[!enabled ? 'hide' : 'show']() + } $('body').on(`click`,`[plugin-manager-action]`,function(e){ e.preventDefault() var el = $(this) @@ -127,6 +147,14 @@ $(document).ready(function(){ } }) break; + case'command': + var command = el.attr('command') + sendInstallerCommand(packageName,command,(data) => { + if(data.ok){ + toggleUsabilityOfYesAndNoButtons(packageName,false) + } + }) + break; } }) $('#downloadNewModule').submit(function(e){ @@ -153,7 +181,7 @@ $(document).ready(function(){ },2000) $.ccio.ws.on('f',function(data){ switch(data.f){ - case'module-info': + case'plugin-info': var name = data.module switch(data.process){ case'install-stdout': @@ -161,6 +189,9 @@ $(document).ready(function(){ // if(loadedBlocks[name].stdout.find('.line').length > 10){ // loadedBlocks[name].stdout.children().first().remove() // } + if(data.data.indexOf('(y)es or (N)o') > -1){ + toggleUsabilityOfYesAndNoButtons(name,true) + } break; case'install-stderr': loadedBlocks[name].stderr.append(`
${data.data}
`) From f9c80277d93e4560e6e7e5cbff96aeb96ceaf681 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Thu, 21 Jan 2021 08:00:52 -0800 Subject: [PATCH 6/7] Plugin Manager : add configuration editor --- languages/en_CA.json | 1 + libs/plugins/superUser.js | 38 +++++++++++++++++++++++++++++- web/libs/js/super.pluginManager.js | 26 ++++++++++++++++++-- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/languages/en_CA.json b/languages/en_CA.json index d13b4a43..c09423ff 100644 --- a/languages/en_CA.json +++ b/languages/en_CA.json @@ -894,6 +894,7 @@ "No Monitor ID Present in Form": "No Monitor ID Present in Form", "State Configuration has no monitors associated": "State Configuration has no monitors associated", "State Configuration Not Found": "State Configuration Not Found", + "Edit Configuration": "Edit Configuration", "Inserted State Configuration": "Inserted State Configuration", "Edited State Configuration": "Edited State Configuration", "Deleted State Configuration": "Deleted State Configuration", diff --git a/libs/plugins/superUser.js b/libs/plugins/superUser.js index e4fae68e..945d08bb 100644 --- a/libs/plugins/superUser.js +++ b/libs/plugins/superUser.js @@ -396,7 +396,43 @@ module.exports = async (s,config,lang,app,io,currentUse) => { runningInstallProcesses[packageName].stdin.write(`${command}\n`) }catch(err){ response.ok = false - response.msg = error + response.msg = err + } + s.closeJsonResponse(res,response) + },res,req) + }) + /** + * API : Superuser : Update Plugin conf.json + */ + app.post(config.webPaths.superApiPrefix+':auth/plugins/configuration/update', (req,res) => { + s.superAuth(req.params, async (resp) => { + const response = {ok: true} + const packageName = req.body.packageName + const configPath = modulesBasePath + packageName + '/conf.json' + const newPluginConfig = s.parseJSON(req.body.config) || {} + try{ + await fs.promises.writeFile(configPath,s.prettyPrint(newPluginConfig)) + }catch(err){ + response.ok = false + response.msg = err + } + s.closeJsonResponse(res,response) + },res,req) + }) + /** + * API : Superuser : Get Plugin conf.json + */ + app.get(config.webPaths.superApiPrefix+':auth/plugins/configuration', (req,res) => { + s.superAuth(req.params, async (resp) => { + const response = {ok: true} + const packageName = req.query.packageName + const modulePath = modulesBasePath + packageName + try{ + const shinobiModule = getModule(packageName) + response.config = shinobiModule.config + }catch(err){ + response.ok = false + response.msg = err } s.closeJsonResponse(res,response) },res,req) diff --git a/web/libs/js/super.pluginManager.js b/web/libs/js/super.pluginManager.js index 13ba571a..3e447d0d 100644 --- a/web/libs/js/super.pluginManager.js +++ b/web/libs/js/super.pluginManager.js @@ -27,6 +27,7 @@ $(document).ready(function(){ ${module.properties.disabled ? lang.Enable : lang.Disable} ` : ''} ${lang.Delete} + ${lang[`Edit Configuration`]}
@@ -111,7 +112,7 @@ $(document).ready(function(){ packageName: packageName, },callback) } - var getPluginBlock = (packageName) => { + var getPluginBlock = function(packageName){ return listElement.find(`[package-name="${packageName}"]`) } var toggleUsabilityOfYesAndNoButtons = function(packageName,enabled){ @@ -149,12 +150,33 @@ $(document).ready(function(){ break; case'command': var command = el.attr('command') - sendInstallerCommand(packageName,command,(data) => { + sendInstallerCommand(packageName,command,function(data){ if(data.ok){ toggleUsabilityOfYesAndNoButtons(packageName,false) } }) break; + case'editConfig': + $.get(superApiPrefix + $user.sessionKey + '/plugins/configuration?packageName=' + packageName,function(data){ + $.confirm.create({ + title: lang[`Edit Configuration`], + body: ``, + clickOptions: { + class: 'btn-success', + title: lang.Save, + }, + clickCallback: function(){ + var newPluginConfigStringed = $('#pluginConfigEditContents').val() + $.post(superApiPrefix + $user.sessionKey + '/plugins/configuration/update',{ + packageName: packageName, + config: newPluginConfigStringed, + },function(data){ + console.log(data) + }) + } + }) + }) + break; } }) $('#downloadNewModule').submit(function(e){ From 18f4bb295eb0d9eadbb2a6590f9e4f1e52253feb Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Thu, 21 Jan 2021 08:32:08 -0800 Subject: [PATCH 7/7] Plugin Manager : allow cancelling install processes --- libs/plugins/superUser.js | 17 ++++---- web/libs/js/super.pluginManager.js | 66 ++++++++++++++++++++++++++---- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/libs/plugins/superUser.js b/libs/plugins/superUser.js index 945d08bb..4909ceac 100644 --- a/libs/plugins/superUser.js +++ b/libs/plugins/superUser.js @@ -146,8 +146,6 @@ module.exports = async (s,config,lang,app,io,currentUse) => { }else if(fs.existsSync(propertiesPath)){ // no INSTALL.sh found, check for package.json and do `npm install --unsafe-perm` installProcess = spawn(`npm`,['install','--unsafe-perm','--prefix',modulePath]) - }else{ - resolve() } if(installProcess){ const sendData = (data,channel) => { @@ -168,10 +166,10 @@ module.exports = async (s,config,lang,app,io,currentUse) => { }) installProcess.on('exit',(data) => { runningInstallProcesses[name] = null; - resolve() }) runningInstallProcesses[name] = installProcess } + resolve() }else{ resolve(lang['Already Installing...']) } @@ -375,11 +373,16 @@ module.exports = async (s,config,lang,app,io,currentUse) => { app.post(config.webPaths.superApiPrefix+':auth/plugins/install', (req,res) => { s.superAuth(req.params, async (resp) => { const packageName = req.body.packageName + const cancelInstall = req.body.cancelInstall === 'true' ? true : false const response = {ok: true} - const error = await installModule(packageName) - if(error){ - response.ok = false - response.msg = error + if(runningInstallProcesses[packageName] && cancelInstall){ + runningInstallProcesses[packageName].kill('SIGTERM') + }else{ + const error = await installModule(packageName) + if(error){ + response.ok = false + response.msg = error + } } s.closeJsonResponse(res,response) },res,req) diff --git a/web/libs/js/super.pluginManager.js b/web/libs/js/super.pluginManager.js index 3e447d0d..34fd41fd 100644 --- a/web/libs/js/super.pluginManager.js +++ b/web/libs/js/super.pluginManager.js @@ -20,12 +20,11 @@ $(document).ready(function(){
${lang['Time Created']} : ${module.created}
${lang['Last Modified']} : ${module.lastModified}
@@ -79,6 +78,8 @@ $(document).ready(function(){ title: lang.Install, }, clickCallback: function(){ + loadedBlocks[packageName].stdout.empty() + loadedBlocks[packageName].stderr.empty() $.post(superApiPrefix + $user.sessionKey + '/plugins/install',{ packageName: packageName, },callback) @@ -113,26 +114,75 @@ $(document).ready(function(){ },callback) } var getPluginBlock = function(packageName){ - return listElement.find(`[package-name="${packageName}"]`) + return loadedBlocks[packageName].block } var toggleUsabilityOfYesAndNoButtons = function(packageName,enabled){ getPluginBlock(packageName).find('.command-installer')[!enabled ? 'hide' : 'show']() } + var toggleCardButtons = function(card,buttons){ + $.each(buttons,function(n,button){ + card.find(`[plugin-manager-action="${button.action}"]`)[button.show ? 'show' : 'hide']() + }) + } $('body').on(`click`,`[plugin-manager-action]`,function(e){ e.preventDefault() var el = $(this) var action = el.attr('plugin-manager-action') var card = el.parents('[package-name]') - console.log(card.length) var packageName = card.attr('package-name') switch(action){ case'install': installModule(packageName,function(data){ if(data.ok){ - console.log(data) + toggleCardButtons(card,[ + { + action: 'install', + show: false, + }, + { + action: 'cancelInstall', + show: true, + }, + { + action: 'delete', + show: false, + }, + { + action: 'status', + show: false, + }, + ]) } }) break; + case'cancelInstall': + $.post(superApiPrefix + $user.sessionKey + '/plugins/install',{ + packageName: packageName, + cancelInstall: 'true' + },function(data){ + if(data.ok){ + toggleCardButtons(card,[ + { + action: 'install', + show: true, + }, + { + action: 'cancelInstall', + show: false, + }, + { + action: 'delete', + show: true, + }, + { + action: 'status', + show: true, + }, + ]) + } + }) + toggleUsabilityOfYesAndNoButtons(packageName,false) + break; case'status': setModuleStatus(packageName,!!!loadedModules[packageName].properties.disabled,function(data){ if(data.ok){