From 4465649c4cbff20c55309c992debf535c954b8ba Mon Sep 17 00:00:00 2001 From: Moe Alam Date: Sun, 18 Oct 2020 21:41:27 -0700 Subject: [PATCH] cleanup cron.js and fix broken knexQuery --- cron.js | 432 +++++++++++++++++++++----------------------- libs/basic/utils.js | 93 ++++++++++ libs/sql/utils.js | 256 ++++++++++++++++++++++++++ 3 files changed, 553 insertions(+), 228 deletions(-) create mode 100644 libs/basic/utils.js create mode 100644 libs/sql/utils.js diff --git a/cron.js b/cron.js index 883b457a..8557700a 100644 --- a/cron.js +++ b/cron.js @@ -1,17 +1,17 @@ process.on('uncaughtException', function (err) { console.error('uncaughtException',err); }); -var fs = require('fs'); -var path = require('path'); -var knex = require('knex'); -var moment = require('moment'); -var exec = require('child_process').exec; -var spawn = require('child_process').spawn; -var config=require('./conf.json'); +const fs = require('fs'); +const path = require('path'); +const moment = require('moment'); +const exec = require('child_process').exec; +const spawn = require('child_process').spawn; +const config = require(process.cwd() + '/conf.json') //set option defaults -s={ - utcOffset : moment().utcOffset() +s = { + mainDirectory: process.cwd(), + utcOffset: moment().utcOffset() }; if(config.cron===undefined)config.cron={}; if(config.cron.deleteOld===undefined)config.cron.deleteOld=true; @@ -29,168 +29,79 @@ if(config.useUTC===undefined){config.useUTC=false} if(config.debugLog===undefined){config.debugLog=false} if(!config.ip||config.ip===''||config.ip.indexOf('0.0.0.0')>-1)config.ip='localhost'; -if(!config.videosDir)config.videosDir=__dirname+'/videos/'; -if(!config.binDir){config.binDir=__dirname+'/fileBin/'} -if(!config.addStorage){config.addStorage=[]} - -// Database Connection -var databaseOptions = { - client: config.databaseType, - connection: config.db, -} -if(databaseOptions.client.indexOf('sqlite')>-1){ - databaseOptions.client = 'sqlite3'; - databaseOptions.useNullAsDefault = true; -} -if(databaseOptions.client === 'sqlite3' && databaseOptions.connection.filename === undefined){ - databaseOptions.connection.filename = __dirname+"/shinobi.sqlite" -} -s.databaseEngine = knex(databaseOptions) -s.dateSubtract = function(date, interval, units){ - var ret = date - var checkRollover = function() { if(ret.getDate() != date.getDate()) ret.setDate(0);}; - switch(interval.toLowerCase()) { - case 'year' : ret.setFullYear(ret.getFullYear() - units); checkRollover(); break; - case 'quarter': ret.setMonth(ret.getMonth() - 3*units); checkRollover(); break; - case 'month' : ret.setMonth(ret.getMonth() - units); checkRollover(); break; - case 'week' : ret.setDate(ret.getDate() - 7*units); break; - case 'day' : ret.setDate(ret.getDate() - units); break; - case 'hour' : ret.setTime(ret.getTime() - units*3600000); break; - case 'minute' : ret.setTime(ret.getTime() - units*60000); break; - case 'second' :default: ret.setTime(ret.getTime() - units*1000); break; - } - return (new Date(ret)) -} -s.sqlDate = function(value){ - var value = value.toLowerCase() - var splitValue = value.split(' ') - var amount = parseFloat(splitValue[0]) - var today = new Date() - var query - if(value.indexOf('min') > -1){ - query = s.dateSubtract(today,'minute',amount) - }else if(value.indexOf('day') > -1){ - query = s.dateSubtract(today,'day',amount) - }else if(value.indexOf('hour') > -1){ - query = s.dateSubtract(today,'hour',amount) - } - return query -} -s.mergeQueryValues = function(query,values){ - if(!values){values=[]} - var valuesNotFunction = true; - if(typeof values === 'function'){ - var values = []; - valuesNotFunction = false; - } - if(values&&valuesNotFunction){ - var splitQuery = query.split('?') - var newQuery = '' - splitQuery.forEach(function(v,n){ - newQuery += v - var value = values[n] - if(value){ - if(isNaN(value) || value instanceof Date){ - newQuery += "'"+value+"'" - }else{ - newQuery += value - } - } - }) - }else{ - newQuery = query - } - return newQuery -} -s.stringToSqlTime = function(value){ - newValue = new Date(value.replace('T',' ')) - return newValue -} -s.sqlQuery = function(query,values,onMoveOn){ - if(!values){values=[]} - if(typeof values === 'function'){ - var onMoveOn = values; - var values = []; - } - if(!onMoveOn){onMoveOn=function(){}} - var mergedQuery = s.mergeQueryValues(query,values) - s.debugLog('s.sqlQuery QUERY',mergedQuery) - return s.databaseEngine - .raw(query,values) - .asCallback(function(err,r){ - if(err){ - console.log('s.sqlQuery QUERY ERRORED',query) - console.log('s.sqlQuery ERROR',err) - } - if(onMoveOn && typeof onMoveOn === 'function'){ - switch(databaseOptions.client){ - case'sqlite3': - if(!r)r=[] - break; - default: - if(r)r=r[0] - break; - } - onMoveOn(err,r) - } - }) -} +if(!config.videosDir)config.videosDir = s.mainDirectory + '/videos/'; +if(!config.binDir){config.binDir = s.mainDirectory + '/fileBin/'} +const { + checkCorrectPathEnding, + generateRandomId, + formattedTime, + localToUtc, +} = require('./libs/basic/utils.js')(s.mainDirectory) +const { + sqlDate, + knexQuery, + initiateDatabaseEngine +} = require('./libs/sql/utils.js')(s,config) +var theCronInterval = null +const overlapLocks = {} +const alreadyDeletedRowsWithNoVideosOnStart = {} +const videoDirectory = checkCorrectPathEnding(config.videosDir) +const fileBinDirectory = checkCorrectPathEnding(config.binDir) s.debugLog = function(arg1,arg2){ if(config.debugLog === true){ if(!arg2)arg2 = '' console.log(arg1,arg2) } } - -//containers -var overlapLocks = {} -s.alreadyDeletedRowsWithNoVideosOnStart={}; -//functions -s.checkCorrectPathEnding=function(x){ - var length=x.length - if(x.charAt(length-1)!=='/'){ - x=x+'/' - } - return x.replace('__DIR__',__dirname) +const connectToMainProcess = () => { + const io = require('socket.io-client')('ws://'+config.ip+':'+config.port,{ + transports:['websocket'] + }); + io.on('connect',function(d){ + postMessage({ + f: 'init', + time: moment() + }) + }) + io.on('f',function(d){ + //command from main process + switch(d.f){ + case'start':case'restart': + setIntervalForCron() + break; + case'stop': + clearCronInterval() + break; + } + }) + return io } -s.dir={ - videos:s.checkCorrectPathEnding(config.videosDir), - fileBin:s.checkCorrectPathEnding(config.binDir), - addStorage:config.addStorage, -}; -s.moment=function(e,x){ - if(!e){e=new Date};if(!x){x='YYYY-MM-DDTHH-mm-ss'}; - return moment(e).format(x); +const postMessage = (x) => { + x.cronKey = config.cron.key; + return io.emit('cron',x) } -s.utcToLocal = function(time){ - return moment.utc(time).utcOffset(s.utcOffset).format() +const sendToWebSocket = (x,y) => { + //emulate master socket emitter + postMessage({f:'s.tx',data:x,to:y}) } -s.localToUtc = function(time){ - return moment(time).utc() +const deleteVideo = (x) => { + postMessage({f:'s.deleteVideo',file:x}) } -s.nameToTime = function(x){x=x.replace('.webm','').replace('.mp4','').split('T'),x[1]=x[1].replace(/-/g,':');x=x.join(' ');return x;} -io = require('socket.io-client')('ws://'+config.ip+':'+config.port,{transports:['websocket']});//connect to master -s.cx = function(x){x.cronKey=config.cron.key;return io.emit('cron',x)} -//emulate master socket emitter -s.tx = function(x,y){s.cx({f:'s.tx',data:x,to:y})} -s.deleteVideo = function(x){s.cx({f:'s.deleteVideo',file:x})} -//Cron Job -s.cx({f:'init',time:moment()}) -s.getVideoDirectory = function(e){ +const getVideoDirectory = function(e){ if(e.mid&&!e.id){e.id=e.mid}; if(e.details&&(e.details instanceof Object)===false){ try{e.details=JSON.parse(e.details)}catch(err){} } if(e.details.dir&&e.details.dir!==''){ - return s.checkCorrectPathEnding(e.details.dir)+e.ke+'/'+e.id+'/' + return checkCorrectPathEnding(e.details.dir)+e.ke+'/'+e.id+'/' }else{ - return s.dir.videos+e.ke+'/'+e.id+'/'; + return videoDirectory + e.ke + '/' + e.id + '/' } } -s.getFileBinDirectory = function(e){ - if(e.mid&&!e.id){e.id=e.mid}; - return s.dir.fileBin+e.ke+'/'+e.id+'/'; +const getFileBinDirectory = function(e){ + if(e.mid && !e.id){e.id = e.mid} + return fileBinDirectory + e.ke + '/' + e.id + '/' } //filters set by the user in their dashboard //deleting old videos is part of the filter - config.cron.deleteOld @@ -204,7 +115,7 @@ const checkFilterRules = function(v,callback){ var where = [{ "p1":"end", "p2":"<=", - "p3":s.sqlDate(v.d.days+" DAY") + "p3": sqlDate(v.d.days+" DAY") }] //exclude monitors with their own max days v.monitorsWithMaxKeepDays.forEach(function(mid){ @@ -236,31 +147,26 @@ const checkFilterRules = function(v,callback){ var b = v.d.filters[m]; s.debugLog(b) if(b.enabled==="1"){ - b.ar=[v.ke]; - b.sql=[]; - b.where.forEach(function(j,k){ - if(j.p1==='ke'){j.p3=v.ke} - switch(j.p3_type){ - case'function': - b.sql.push(j.p1+' '+j.p2+' '+j.p3) - break; - default: - b.sql.push(j.p1+' '+j.p2+' ?') - b.ar.push(j.p3) - break; - } + const whereQuery = [ + ['ke','=',v.ke], + ['status','!=',"0"], + ['details','NOT LIKE','%"archived":"1"%'], + ] + b.where.forEach(function(condition){ + if(condition.p1 === 'ke'){condition.p3 = v.ke} + whereQuery.push([condition.p1,condition.p2 || '=',condition.p3]) }) - b.sql='WHERE ke=? AND status != 0 AND details NOT LIKE \'%"archived":"1"%\' AND ('+b.sql.join(' AND ')+')'; - if(b.sort_by&&b.sort_by!==''){ - b.sql+=' ORDER BY `'+b.sort_by+'` '+b.sort_by_direction - } - if(b.limit&&b.limit!==''){ - b.sql+=' LIMIT '+b.limit - } - s.sqlQuery('SELECT * FROM Videos '+b.sql,b.ar,function(err,r){ - if(r&&r[0]){ + knexQuery({ + action: "select", + columns: "*", + table: "Videos", + where: whereQuery, + orderBy: [b.sort_by,b.sort_by_direction.toLowerCase()], + limit: b.limit + },(err,r) => { + if(r && r[0]){ if(r.length > 0 || config.debugLog === true){ - s.cx({f:'filterMatch',msg:r.length+' SQL rows match "'+m+'"',ke:v.ke,time:moment()}) + postMessage({f:'filterMatch',msg:r.length+' SQL rows match "'+m+'"',ke:v.ke,time:moment()}) } b.cx={ f:'filters', @@ -271,9 +177,9 @@ const checkFilterRules = function(v,callback){ id:b.id }; if(b.archive==="1"){ - s.cx({f:'filters',ff:'archive',videos:r,time:moment(),ke:v.ke,id:b.id}); + postMessage({f:'filters',ff:'archive',videos:r,time:moment(),ke:v.ke,id:b.id}); }else if(b.delete==="1"){ - s.cx({f:'filters',ff:'delete',videos:r,time:moment(),ke:v.ke,id:b.id}); + postMessage({f:'filters',ff:'delete',videos:r,time:moment(),ke:v.ke,id:b.id}); } if(b.email==="1"){ b.cx.ff='email'; @@ -281,10 +187,10 @@ const checkFilterRules = function(v,callback){ b.cx.mail=v.mail; b.cx.execute=b.execute; b.cx.query=b.sql; - s.cx(b.cx); + postMessage(b.cx); } if(b.execute&&b.execute!==""){ - s.cx({f:'filters',ff:'execute',execute:b.execute,time:moment()}); + postMessage({f:'filters',ff:'execute',execute:b.execute,time:moment()}); } } }) @@ -305,14 +211,23 @@ const deleteRowsWithNoVideo = function(v,callback){ if( config.cron.deleteNoVideo===true&&( config.cron.deleteNoVideoRecursion===true|| - (config.cron.deleteNoVideoRecursion===false&&!s.alreadyDeletedRowsWithNoVideosOnStart[v.ke]) + (config.cron.deleteNoVideoRecursion===false&&!alreadyDeletedRowsWithNoVideosOnStart[v.ke]) ) ){ - s.alreadyDeletedRowsWithNoVideosOnStart[v.ke]=true; - es={}; - s.sqlQuery('SELECT * FROM Videos WHERE ke=? AND status!=0 AND details NOT LIKE \'%"archived":"1"%\' AND time < ?',[v.ke,s.sqlDate('10 MINUTE')],function(err,evs){ - if(evs&&evs[0]){ - es.del=[];es.ar=[v.ke]; + alreadyDeletedRowsWithNoVideosOnStart[v.ke]=true; + knexQuery({ + action: "select", + columns: "*", + table: "Videos", + where: [ + ['ke','=',v.ke], + ['status','!=','0'], + ['details','NOT LIKE','%"archived":"1"%'], + ['time','<', sqlDate('10 MINUTE')], + ] + },(err,evs) => { + if(evs && evs[0]){ + const videosToDelete = []; evs.forEach(function(ev){ var filename var details @@ -325,20 +240,20 @@ const deleteRowsWithNoVideo = function(v,callback){ details = {} } } - var dir = s.getVideoDirectory(ev) + var dir = getVideoDirectory(ev) if(details.isUTC === true){ - filename = s.localToUtc(ev.time).format('YYYY-MM-DDTHH-mm-ss')+'.'+ev.ext + filename = localToUtc(ev.time).format('YYYY-MM-DDTHH-mm-ss')+'.'+ev.ext }else{ - filename = s.moment(ev.time)+'.'+ev.ext + filename = formattedTime(ev.time)+'.'+ev.ext } fileExists = fs.existsSync(dir+filename) if(fileExists !== true){ - s.deleteVideo(ev) - s.tx({f:'video_delete',filename:filename+'.'+ev.ext,mid:ev.mid,ke:ev.ke,time:ev.time,end:s.moment(new Date,'YYYY-MM-DD HH:mm:ss')},'GRP_'+ev.ke); + deleteVideo(ev) + sendToWebSocket({f:'video_delete',filename:filename+'.'+ev.ext,mid:ev.mid,ke:ev.ke,time:ev.time,end: formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')},'GRP_'+ev.ke); } }); - if(es.del.length>0 || config.debugLog === true){ - s.cx({f:'deleteNoVideo',msg:es.del.length+' SQL rows with no file deleted',ke:v.ke,time:moment()}) + if(videosToDelete.length>0 || config.debugLog === true){ + postMessage({f:'deleteNoVideo',msg:videosToDelete.length+' SQL rows with no file deleted',ke:v.ke,time:moment()}) } } setTimeout(function(){ @@ -353,11 +268,18 @@ const deleteRowsWithNoVideo = function(v,callback){ const deleteOldLogs = function(v,callback){ if(!v.d.log_days||v.d.log_days==''){v.d.log_days=10}else{v.d.log_days=parseFloat(v.d.log_days)}; if(config.cron.deleteLogs===true&&v.d.log_days!==0){ - s.sqlQuery("DELETE FROM Logs WHERE ke=? AND `time` < ?",[v.ke,s.sqlDate(v.d.log_days+' DAY')],function(err,rrr){ + knexQuery({ + action: "delete", + table: "Logs", + where: [ + ['ke','=',v.ke], + ['time','<', sqlDate(v.d.log_days+' DAY')], + ] + },(err,rrr) => { callback() if(err)return console.error(err); if(rrr.affectedRows && rrr.affectedRows.length>0 || config.debugLog === true){ - s.cx({f:'deleteLogs',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.log_days+' days deleted',ke:v.ke,time:moment()}) + postMessage({f:'deleteLogs',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.log_days+' days deleted',ke:v.ke,time:moment()}) } }) }else{ @@ -368,11 +290,40 @@ const deleteOldLogs = function(v,callback){ const deleteOldEvents = function(v,callback){ if(!v.d.event_days||v.d.event_days==''){v.d.event_days=10}else{v.d.event_days=parseFloat(v.d.event_days)}; if(config.cron.deleteEvents===true&&v.d.event_days!==0){ - s.sqlQuery("DELETE FROM Events WHERE ke=? AND `time` < ?",[v.ke,s.sqlDate(v.d.event_days+' DAY')],function(err,rrr){ + knexQuery({ + action: "delete", + table: "Events", + where: [ + ['ke','=',v.ke], + ['time','<', sqlDate(v.d.event_days+' DAY')], + ] + },(err,rrr) => { callback() if(err)return console.error(err); - if(rrr.affectedRows && rrr.affectedRows.length>0 || config.debugLog === true){ - s.cx({f:'deleteEvents',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.event_days+' days deleted',ke:v.ke,time:moment()}) + if(rrr.affectedRows && rrr.affectedRows.length > 0 || config.debugLog === true){ + postMessage({f:'deleteEvents',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.event_days+' days deleted',ke:v.ke,time:moment()}) + } + }) + }else{ + callback() + } +} +//event counts +const deleteOldEventCounts = function(v,callback){ + if(!v.d.event_days||v.d.event_days==''){v.d.event_days=10}else{v.d.event_days=parseFloat(v.d.event_days)}; + if(config.cron.deleteEvents===true&&v.d.event_days!==0){ + knexQuery({ + action: "delete", + table: "Events Counts", + where: [ + ['ke','=',v.ke], + ['time','<', sqlDate(v.d.event_days+' DAY')], + ] + },(err,rrr) => { + callback() + if(err && err.code !== 'ER_NO_SUCH_TABLE')return console.error(err); + if(rrr.affectedRows && rrr.affectedRows.length > 0 || config.debugLog === true){ + postMessage({f:'deleteEvents',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.event_days+' days deleted',ke:v.ke,time:moment()}) } }) }else{ @@ -384,20 +335,35 @@ const deleteOldFileBins = function(v,callback){ if(!v.d.fileBin_days||v.d.fileBin_days==''){v.d.fileBin_days=10}else{v.d.fileBin_days=parseFloat(v.d.fileBin_days)}; if(config.cron.deleteFileBins===true&&v.d.fileBin_days!==0){ var fileBinQuery = " FROM Files WHERE ke=? AND `time` < ?"; - s.sqlQuery("SELECT *"+fileBinQuery,[v.ke,s.sqlDate(v.d.fileBin_days+' DAY')],function(err,files){ + knexQuery({ + action: "select", + columns: "*", + table: "Files", + where: [ + ['ke','=',v.ke], + ['time','<', sqlDate(v.d.fileBin_days+' DAY')], + ] + },(err,files) => { if(files&&files[0]){ //delete the files files.forEach(function(file){ - fs.unlink(s.getFileBinDirectory(file)+file.name,function(err){ + fs.unlink(getFileBinDirectory(file) + file.name,function(err){ // if(err)console.error(err) }) }) //delete the database rows - s.sqlQuery("DELETE"+fileBinQuery,[v.ke,v.d.fileBin_days],function(err,rrr){ + knexQuery({ + action: "delete", + table: "Files", + where: [ + ['ke','=',v.ke], + ['time','<', sqlDate(v.d.fileBin_days+' DAY')], + ] + },(err,rrr) => { callback() if(err)return console.error(err); if(rrr.affectedRows && rrr.affectedRows.length>0 || config.debugLog === true){ - s.cx({f:'deleteFileBins',msg:(rrr.affectedRows || 0)+' files older than '+v.d.fileBin_days+' days deleted',ke:v.ke,time:moment()}) + postMessage({f:'deleteFileBins',msg:(rrr.affectedRows || 0)+' files older than '+v.d.fileBin_days+' days deleted',ke:v.ke,time:moment()}) } }) }else{ @@ -424,8 +390,8 @@ const processUser = function(number,rows){ return } s.debugLog(v) - if(!s.alreadyDeletedRowsWithNoVideosOnStart[v.ke]){ - s.alreadyDeletedRowsWithNoVideosOnStart[v.ke]=false; + if(!alreadyDeletedRowsWithNoVideosOnStart[v.ke]){ + alreadyDeletedRowsWithNoVideosOnStart[v.ke]=false; } if(!overlapLocks[v.ke]){ // set overlap lock @@ -436,7 +402,14 @@ const processUser = function(number,rows){ if(!v.d.size||v.d.size==''){v.d.size=10000}else{v.d.size=parseFloat(v.d.size)}; //days to keep videos if(!v.d.days||v.d.days==''){v.d.days=5}else{v.d.days=parseFloat(v.d.days)}; - s.sqlQuery('SELECT * FROM Monitors WHERE ke=?', [v.ke], function(err,rr) { + knexQuery({ + action: "select", + columns: "*", + table: "Monitors", + where: [ + ['ke','=',v.ke], + ] + },(err,rr) => { if(!v.d.filters||v.d.filters==''){ v.d.filters={}; } @@ -463,7 +436,7 @@ const processUser = function(number,rows){ },{ "p1":"end", "p2":"<", - "p3":s.sqlDate(b.details.max_keep_days+" DAY") + "p3": sqlDate(b.details.max_keep_days+" DAY") }] }; } @@ -474,14 +447,17 @@ const processUser = function(number,rows){ s.debugLog('--- deleteOldFileBins Complete') deleteOldEvents(v,function(){ s.debugLog('--- deleteOldEvents Complete') - checkFilterRules(v,function(){ - s.debugLog('--- checkFilterRules Complete') - deleteRowsWithNoVideo(v,function(){ - s.debugLog('--- deleteRowsWithNoVideo Complete') - checkForOrphanedFiles(v,function(){ - //done user, unlock current, and do next - overlapLocks[v.ke]=false; - processUser(number+1,rows) + deleteOldEventCounts(v,function(){ + s.debugLog('--- deleteOldEventCounts Complete') + checkFilterRules(v,function(){ + s.debugLog('--- checkFilterRules Complete') + deleteRowsWithNoVideo(v,function(){ + s.debugLog('--- deleteRowsWithNoVideo Complete') + checkForOrphanedFiles(v,function(){ + //done user, unlock current, and do next + overlapLocks[v.ke]=false; + processUser(number+1,rows) + }) }) }) }) @@ -494,7 +470,6 @@ const processUser = function(number,rows){ } } //recursive function -var theCronInterval = null const setIntervalForCron = function(){ clearCronInterval() theCronInterval = setInterval(doCronJobs,parseFloat(config.cron.interval)*60000*60) @@ -503,8 +478,18 @@ const clearCronInterval = function(){ clearInterval(theCronInterval) } const doCronJobs = function(){ - s.cx({f:'start',time:moment()}) - s.sqlQuery('SELECT ke,uid,details,mail FROM Users WHERE details NOT LIKE \'%"sub"%\'', function(err,rows) { + postMessage({ + f: 'start', + time: moment() + }) + knexQuery({ + action: "select", + columns: "ke,uid,details,mail", + table: "Users", + where: [ + ['details','NOT LIKE','%"sub"%'], + ] + },(err,rows) => { if(err){ console.error(err) } @@ -513,17 +498,8 @@ const doCronJobs = function(){ } }) } +initiateDatabaseEngine() +const io = connectToMainProcess() setIntervalForCron() doCronJobs() -//socket commander -io.on('f',function(d){ - switch(d.f){ - case'start':case'restart': - setIntervalForCron() - break; - case'stop': - clearCronInterval() - break; - } -}) console.log('Shinobi : cron.js started') diff --git a/libs/basic/utils.js b/libs/basic/utils.js new file mode 100644 index 00000000..33dd4d4f --- /dev/null +++ b/libs/basic/utils.js @@ -0,0 +1,93 @@ +const fs = require('fs'); +const path = require('path'); +const moment = require('moment'); +module.exports = (processCwd) => { + const parseJSON = (string) => { + var parsed + try{ + parsed = JSON.parse(string) + }catch(err){ + + } + if(!parsed)parsed = string + return parsed + } + const stringJSON = (json) => { + try{ + if(json instanceof Object){ + json = JSON.stringify(json) + } + }catch(err){ + + } + return json + } + const stringContains = (find,string,toLowerCase) => { + var newString = string + '' + if(toLowerCase)newString = newString.toLowerCase() + return newString.indexOf(find) > -1 + } + const checkCorrectPathEnding = (x) => { + var length=x.length + if(x.charAt(length-1)!=='/'){ + x=x+'/' + } + return x.replace('__DIR__',processCwd) + } + const mergeDeep = function(...objects) { + const isObject = obj => obj && typeof obj === 'object'; + + return objects.reduce((prev, obj) => { + Object.keys(obj).forEach(key => { + const pVal = prev[key]; + const oVal = obj[key]; + + if (Array.isArray(pVal) && Array.isArray(oVal)) { + prev[key] = pVal.concat(...oVal); + } + else if (isObject(pVal) && isObject(oVal)) { + prev[key] = mergeDeep(pVal, oVal); + } + else { + prev[key] = oVal; + } + }); + + return prev; + }, {}); + } + const nameToTime = (x) => { + x = x.split('.')[0].split('T') + if(x[1])x[1] = x[1].replace(/-/g,':') + x = x.join(' ') + return x + } + const generateRandomId = (x) => { + if(!x){x=10};var t = "";var p = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + for( var i=0; i < x; i++ ) + t += p.charAt(Math.floor(Math.random() * p.length)); + return t; + } + const utcToLocal = (time) => { + return moment.utc(time).utcOffset(s.utcOffset).format() + } + const localToUtc = (time) => { + return moment(time).utc() + } + const formattedTime = (e,x) => { + if(!e){e=new Date};if(!x){x='YYYY-MM-DDTHH-mm-ss'}; + return moment(e).format(x); + } + return { + parseJSON: parseJSON, + stringJSON: stringJSON, + stringContains: stringContains, + checkCorrectPathEnding: checkCorrectPathEnding, + nameToTime: nameToTime, + mergeDeep: mergeDeep, + generateRandomId: generateRandomId, + utcToLocal: utcToLocal, + localToUtc: localToUtc, + formattedTime: formattedTime, + } +} diff --git a/libs/sql/utils.js b/libs/sql/utils.js new file mode 100644 index 00000000..2cd0d9dd --- /dev/null +++ b/libs/sql/utils.js @@ -0,0 +1,256 @@ +var knex = require('knex'); +module.exports = (s,config,databaseOptions) => { + var databaseOptions = { + client: config.databaseType, + connection: config.db, + } + if(databaseOptions.client.indexOf('sqlite')>-1){ + databaseOptions.client = 'sqlite3'; + databaseOptions.useNullAsDefault = true; + } + if(databaseOptions.client === 'sqlite3' && databaseOptions.connection.filename === undefined){ + databaseOptions.connection.filename = __dirname+"/shinobi.sqlite" + } + const dateSubtract = function(date, interval, units){ + var ret = date + var checkRollover = function() { if(ret.getDate() != date.getDate()) ret.setDate(0);}; + switch(interval.toLowerCase()) { + case 'year' : ret.setFullYear(ret.getFullYear() - units); checkRollover(); break; + case 'quarter': ret.setMonth(ret.getMonth() - 3*units); checkRollover(); break; + case 'month' : ret.setMonth(ret.getMonth() - units); checkRollover(); break; + case 'week' : ret.setDate(ret.getDate() - 7*units); break; + case 'day' : ret.setDate(ret.getDate() - units); break; + case 'hour' : ret.setTime(ret.getTime() - units*3600000); break; + case 'minute' : ret.setTime(ret.getTime() - units*60000); break; + case 'second' :default: ret.setTime(ret.getTime() - units*1000); break; + } + return (new Date(ret)) + } + const sqlDate = function(value){ + var value = value.toLowerCase() + var splitValue = value.split(' ') + var amount = parseFloat(splitValue[0]) + var today = new Date() + var query + if(value.indexOf('min') > -1){ + query = dateSubtract(today,'minute',amount) + }else if(value.indexOf('day') > -1){ + query = dateSubtract(today,'day',amount) + }else if(value.indexOf('hour') > -1){ + query = dateSubtract(today,'hour',amount) + } + return query + } + const mergeQueryValues = function(query,values){ + if(!values){values=[]} + var valuesNotFunction = true; + if(typeof values === 'function'){ + var values = []; + valuesNotFunction = false; + } + if(values&&valuesNotFunction){ + var splitQuery = query.split('?') + var newQuery = '' + splitQuery.forEach(function(v,n){ + newQuery += v + var value = values[n] + if(value){ + if(isNaN(value) || value instanceof Date){ + newQuery += "'"+value+"'" + }else{ + newQuery += value + } + } + }) + }else{ + newQuery = query + } + return newQuery + } + const stringToSqlTime = function(value){ + newValue = new Date(value.replace('T',' ')) + return newValue + } + const sqlQuery = function(query,values,onMoveOn){ + if(!values){values=[]} + if(typeof values === 'function'){ + var onMoveOn = values; + var values = []; + } + if(!onMoveOn){onMoveOn=function(){}} + var mergedQuery = mergeQueryValues(query,values) + s.debugLog('s.sqlQuery QUERY',mergedQuery) + return s.databaseEngine + .raw(query,values) + .asCallback(function(err,r){ + if(err){ + console.log('s.sqlQuery QUERY ERRORED',query) + console.log('s.sqlQuery ERROR',err) + } + if(onMoveOn && typeof onMoveOn === 'function'){ + switch(databaseOptions.client){ + case'sqlite3': + if(!r)r=[] + break; + default: + if(r)r=r[0] + break; + } + onMoveOn(err,r) + } + }) + } + const cleanSqlWhereObject = (where) => { + const newWhere = {} + Object.keys(where).forEach((key) => { + if(key !== '__separator'){ + const value = where[key] + newWhere[key] = value + } + }) + return newWhere + } + const processSimpleWhereCondition = (dbQuery,where,didOne) => { + var whereIsArray = where instanceof Array; + if(where[0] === 'or' || where.__separator === 'or'){ + if(whereIsArray){ + where.shift() + dbQuery.orWhere(...where) + }else{ + where = cleanSqlWhereObject(where) + dbQuery.orWhere(where) + } + }else if(!didOne){ + didOne = true + whereIsArray ? dbQuery.where(...where) : dbQuery.where(where) + }else{ + whereIsArray ? dbQuery.andWhere(...where) : dbQuery.andWhere(where) + } + } + const processWhereCondition = (dbQuery,where,didOne) => { + var whereIsArray = where instanceof Array; + if(!where[0])return; + if(where[0] && where[0] instanceof Array){ + dbQuery.where(function() { + var _this = this + var didOneInsideGroup = false + where.forEach((whereInsideGroup) => { + processWhereCondition(_this,whereInsideGroup,didOneInsideGroup) + }) + }) + }else if(where[0] && where[0] instanceof Object){ + dbQuery.where(function() { + var _this = this + var didOneInsideGroup = false + where.forEach((whereInsideGroup) => { + processSimpleWhereCondition(_this,whereInsideGroup,didOneInsideGroup) + }) + }) + }else{ + processSimpleWhereCondition(dbQuery,where,didOne) + } + } + const knexError = (dbQuery,options,err) => { + console.error('knexError----------------------------------- START') + if(config.debugLogVerbose && config.debugLog === true){ + s.debugLog('s.knexQuery QUERY',JSON.stringify(options,null,3)) + s.debugLog('STACK TRACE, NOT AN ',new Error()) + } + console.error(err) + console.error(dbQuery.toString()) + console.error('knexError----------------------------------- END') + } + const knexQuery = (options,callback) => { + try{ + if(!s.databaseEngine)return// console.log('Database Not Set'); + // options = { + // action: "", + // columns: "", + // table: "" + // } + var dbQuery + switch(options.action){ + case'select': + options.columns = options.columns.indexOf(',') === -1 ? [options.columns] : options.columns.split(','); + dbQuery = s.databaseEngine.select(...options.columns).from(options.table) + break; + case'count': + options.columns = options.columns.indexOf(',') === -1 ? [options.columns] : options.columns.split(','); + dbQuery = s.databaseEngine(options.table) + dbQuery.count(options.columns) + break; + case'update': + dbQuery = s.databaseEngine(options.table).update(options.update) + break; + case'delete': + dbQuery = s.databaseEngine(options.table) + break; + case'insert': + dbQuery = s.databaseEngine(options.table).insert(options.insert) + break; + } + if(options.where instanceof Array){ + var didOne = false; + options.where.forEach((where) => { + processWhereCondition(dbQuery,where,didOne) + }) + }else if(options.where instanceof Object){ + dbQuery.where(options.where) + } + if(options.action === 'delete'){ + dbQuery.del() + } + if(options.orderBy){ + dbQuery.orderBy(...options.orderBy) + } + if(options.groupBy){ + dbQuery.groupBy(options.groupBy) + } + if(options.limit){ + if(`${options.limit}`.indexOf(',') === -1){ + dbQuery.limit(options.limit) + }else{ + const limitParts = `${options.limit}`.split(',') + dbQuery.limit(limitParts[1]).offset(limitParts[0]) + } + } + if(config.debugLog === true){ + console.log(dbQuery.toString()) + } + if(callback || options.update || options.insert || options.action === 'delete'){ + dbQuery.asCallback(function(err,r) { + if(err){ + knexError(dbQuery,options,err) + } + if(callback)callback(err,r) + if(config.debugLogVerbose && config.debugLog === true){ + s.debugLog('s.knexQuery QUERY',JSON.stringify(options,null,3)) + s.debugLog('s.knexQuery RESPONSE',JSON.stringify(r,null,3)) + s.debugLog('STACK TRACE, NOT AN ',new Error()) + } + }) + } + return dbQuery + }catch(err){ + if(callback)callback(err,[]) + knexError(dbQuery,options,err) + } + } + const initiateDatabaseEngine = () => { + s.databaseEngine = knex(databaseOptions) + return s.databaseEngine + } + return { + dateSubtract: dateSubtract, + sqlDate: sqlDate, + mergeQueryValues: mergeQueryValues, + stringToSqlTime: stringToSqlTime, + sqlQuery: sqlQuery, + cleanSqlWhereObject: cleanSqlWhereObject, + processSimpleWhereCondition: processSimpleWhereCondition, + processWhereCondition: processWhereCondition, + knexError: knexError, + knexQuery: knexQuery, + initiateDatabaseEngine: initiateDatabaseEngine + } +}