From e791a9895eb59c28278f380678a643c8f754fc75 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Sun, 6 Sep 2020 09:41:51 -0700 Subject: [PATCH 1/3] fix deleting monitorStates --- libs/webServerAdminPaths.js | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/webServerAdminPaths.js b/libs/webServerAdminPaths.js index 856202ed..986fc85b 100644 --- a/libs/webServerAdminPaths.js +++ b/libs/webServerAdminPaths.js @@ -555,7 +555,6 @@ module.exports = function(s,config,lang,app){ s.knexQuery({ action: "delete", table: "Presets", - update: monitorQuery, where: { ke: req.params.ke, name: req.params.stateName, From d1e68ffe373b5c53c8eac01d8255e5fa149d9e58 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Sun, 6 Sep 2020 11:55:42 -0700 Subject: [PATCH 2/3] Revert ftp daemon back to ftp-srv - The telnet vulnerability was fixed, no longer any need to attempt migration --- libs/dropInEvents.js | 107 +++++++++++++------------------------------ package.json | 2 +- 2 files changed, 34 insertions(+), 75 deletions(-) diff --git a/libs/dropInEvents.js b/libs/dropInEvents.js index d11ef50b..50d8b7af 100644 --- a/libs/dropInEvents.js +++ b/libs/dropInEvents.js @@ -183,87 +183,46 @@ module.exports = function(s,config,lang,app,io){ } // FTP Server if(config.ftpServer === true){ - const authenticateConnection = (connection) => { - return new Promise((resolve,reject) => { - var username = null; - s.debugLog('client connected: ' + connection.remoteAddress); - connection.on('command:user', function(user, success, failure) { - if (user) { - username = user; - success(); - } else { - failure(); - } - }) - - connection.on('command:pass', function(password, success, failure) { - s.basicOrApiAuthentication(username,password,function(err,user){ - if(user){ - connection._user = user - success(username); - } else { - failure(); - } - resolve({ - ok: !!user, - username: username, - password: password - }) - }) - }) - }) - } createDropInEventsDirectory() if(!config.ftpServerPort)config.ftpServerPort = 21 if(!config.ftpServerUrl)config.ftpServerUrl = `ftp://0.0.0.0:${config.ftpServerPort}` config.ftpServerUrl = config.ftpServerUrl.replace('{{PORT}}',config.ftpServerPort) - const ftpd = require('shinobi-ftpd') - const ftpServer = new ftpd.FtpServer(config.ftpServerUrl, Object.assign({ - getInitialCwd: function(connection, callback) { - callback(null, s.dir.dropInEvents + '/' + connection._user.ke) - }, - getRoot: function() { - return s.dir.dropInEvents - }, - pasvPortRangeStart: 1025, - pasvPortRangeEnd: 1050, - allowUnauthorizedTls: true, - uploadMaxSlurpSize: 7000 - },config.ftpServerOptions || {})) - - ftpServer.on('error', function(error) { - s.debugLog(['FTP Server error:', error]); + const FtpSrv = require('ftp-srv') + const ftpServer = new FtpSrv({ + url: config.ftpServerUrl, + log: require('bunyan').createLogger({ + name: 'ftp-srv', + level: 100 + }), }) - ftpServer.on('client:connected', async (connection) => { - const response = await authenticateConnection(connection) - if(connection._user){ - connection.cwd = s.dir.dropInEvents + connection._user.ke - connection.on('file:stor', async (eventType, data) => { - const fileName = data.file - const pathPieces = fileName.substr(1).split('/') - const groupKey = connection._user.ke - const monitorId = pathPieces[0] - const firstDroppedPart = pathPieces[1] - const monitorEventDropDir = s.dir.dropInEvents + groupKey + '/' + monitorId + '/' - const deleteKey = monitorEventDropDir + firstDroppedPart - onFileOrFolderFound( - monitorEventDropDir + firstDroppedPart, - deleteKey, - { - ke: groupKey, - mid: monitorId - } - ) - }) - }else{ - s.systemLog(`Failed FTP Login Attempt : ${response.username}/${response.password}`) - throw `Failed to Authenticate FTP : ${response.username}/${response.password}`; - } + ftpServer.on('login', ({connection, username, password}, resolve, reject) => { + s.basicOrApiAuthentication(username,password,function(err,user){ + if(user){ + connection.on('STOR', (error, fileName) => { + if(!fileName)return; + var pathPieces = fileName.replace(s.dir.dropInEvents,'').split('/') + var ke = pathPieces[0] + var mid = pathPieces[1] + var firstDroppedPart = pathPieces[2] + var monitorEventDropDir = s.dir.dropInEvents + ke + '/' + mid + '/' + var deleteKey = monitorEventDropDir + firstDroppedPart + onFileOrFolderFound(monitorEventDropDir + firstDroppedPart,deleteKey,{ke: ke, mid: mid}) + }) + resolve({root: s.dir.dropInEvents + user.ke}) + }else{ + // reject(new Error('Failed Authorization')) + } + }) + }) + ftpServer.on('client-error', ({connection, context, error}) => { + console.log('client-error',error) + }) + ftpServer.listen().then(() => { + s.systemLog(`FTP Server running on port ${config.ftpServerPort}...`) + }).catch(function(err){ + s.systemLog(err) }) - - ftpServer.listen(config.ftpServerPort) - s.systemLog(`FTP Server running on port ${config.ftpServerPort}...`) } //add extensions s.onMonitorInit(onMonitorInit) diff --git a/package.json b/package.json index 689a61c7..4f9d5598 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "discord.js": "^12.2.0", "ejs": "^2.5.5", "express": "^4.16.4", - "shinobi-ftpd": "0.2.18", + "ftp-srv": "4.3.4", "http-proxy": "^1.17.0", "jsonfile": "^3.0.1", "knex": "^0.21.4", From 22c9b334b6dbdf84eca019c5dc7db23ff82db9d4 Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Sun, 6 Sep 2020 12:11:36 -0700 Subject: [PATCH 3/3] Cache video row when viewing to prevent database hammer on chunking --- libs/webServerPaths.js | 72 +++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/libs/webServerPaths.js b/libs/webServerPaths.js index f5c89102..0a260f74 100644 --- a/libs/webServerPaths.js +++ b/libs/webServerPaths.js @@ -1429,6 +1429,8 @@ module.exports = function(s,config,lang,app,io){ /** * API : Get Video File */ + const videoRowCaches = {} + const videoRowCacheTimeouts = {} app.get(config.webPaths.apiPrefix+':auth/videos/:ke/:id/:file', function (req,res){ s.auth(req.params,function(user){ const groupKey = req.params.ke @@ -1446,34 +1448,52 @@ module.exports = function(s,config,lang,app,io){ time = s.utcToLocal(time) } time = new Date(time) - s.knexQuery({ - action: "select", - columns: "*", - table: "Videos", - where: [ - ['ke','=',groupKey], - ['mid','=',req.params.id], - ['time','=',time] - ], - limit: 1 - },(err,r) => { - if(r&&r[0]){ - req.dir=s.getVideoDirectory(r[0])+req.params.file - fs.stat(req.dir,function(err,stats){ - if (!err){ - if(req.query.json === 'true'){ - s.closeJsonResponse(res,r[0]) - }else{ - s.streamMp4FileOverHttp(req.dir,req,res) - } + const cacheName = Object.values(req.params).join('_') + const cacheVideoRow = (videoRow) => { + videoRowCaches[cacheName] = videoRow + clearTimeout(videoRowCacheTimeouts[cacheName]) + videoRowCacheTimeouts[cacheName] = setTimeout(() => { + console.log('clear cache',cacheName) + delete(videoRowCaches[cacheName]) + },60000) + } + const sendVideo = (videoRow) => { + cacheVideoRow(videoRow) + const filePath = s.getVideoDirectory(videoRow) + req.params.file + fs.stat(filePath,function(err,stats){ + if (!err){ + if(req.query.json === 'true'){ + s.closeJsonResponse(res,videoRow) }else{ - res.end(user.lang['File Not Found in Filesystem']) + s.streamMp4FileOverHttp(filePath,req,res) } - }) - }else{ - res.end(user.lang['File Not Found in Database']) - } - }) + }else{ + res.end(user.lang['File Not Found in Filesystem']) + } + }) + } + if(videoRowCaches[cacheName]){ + sendVideo(videoRowCaches[cacheName]) + }else{ + s.knexQuery({ + action: "select", + columns: "*", + table: "Videos", + where: [ + ['ke','=',groupKey], + ['mid','=',req.params.id], + ['time','=',time] + ], + limit: 1 + },(err,r) => { + const videoRow = r[0] + if(videoRow){ + sendVideo(videoRow) + }else{ + res.end(user.lang['File Not Found in Database']) + } + }) + } },res,req); }); /**