ShinobiSystems
/
Shinobi
mirror of https://gitlab.com/Shinobi-Systems/Shinobi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cleanup cron.js and fix broken knexQuery

merge-requests/289/head
Moe Alam 2020-10-18 21:41:27 -07:00
parent c69d761009
commit 4465649c4c
3 changed files with 553 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

432
cron.js
View File

@ -1,17 +1,17 @@
process.on('uncaughtException', function (err) {
console.error('uncaughtException',err);
});
var fs = require('fs');
var path = require('path');
var knex = require('knex');
var moment = require('moment');
var exec = require('child_process').exec;
var spawn = require('child_process').spawn;
var config=require('./conf.json');
const fs = require('fs');
const path = require('path');
const moment = require('moment');
const exec = require('child_process').exec;
const spawn = require('child_process').spawn;
const config = require(process.cwd() + '/conf.json')
//set option defaults
s={
utcOffset : moment().utcOffset()
s = {
mainDirectory: process.cwd(),
utcOffset: moment().utcOffset()
};
if(config.cron===undefined)config.cron={};
if(config.cron.deleteOld===undefined)config.cron.deleteOld=true;
@ -29,168 +29,79 @@ if(config.useUTC===undefined){config.useUTC=false}
if(config.debugLog===undefined){config.debugLog=false}
if(!config.ip||config.ip===''||config.ip.indexOf('0.0.0.0')>-1)config.ip='localhost';
if(!config.videosDir)config.videosDir=__dirname+'/videos/';
if(!config.binDir){config.binDir=__dirname+'/fileBin/'}
if(!config.addStorage){config.addStorage=[]}
// Database Connection
var databaseOptions = {
client: config.databaseType,
connection: config.db,
}
if(databaseOptions.client.indexOf('sqlite')>-1){
databaseOptions.client = 'sqlite3';
databaseOptions.useNullAsDefault = true;
}
if(databaseOptions.client === 'sqlite3' && databaseOptions.connection.filename === undefined){
databaseOptions.connection.filename = __dirname+"/shinobi.sqlite"
}
s.databaseEngine = knex(databaseOptions)
s.dateSubtract = function(date, interval, units){
var ret = date
var checkRollover = function() { if(ret.getDate() != date.getDate()) ret.setDate(0);};
switch(interval.toLowerCase()) {
case 'year' : ret.setFullYear(ret.getFullYear() - units); checkRollover(); break;
case 'quarter': ret.setMonth(ret.getMonth() - 3*units); checkRollover(); break;
case 'month' : ret.setMonth(ret.getMonth() - units); checkRollover(); break;
case 'week' : ret.setDate(ret.getDate() - 7*units); break;
case 'day' : ret.setDate(ret.getDate() - units); break;
case 'hour' : ret.setTime(ret.getTime() - units*3600000); break;
case 'minute' : ret.setTime(ret.getTime() - units*60000); break;
case 'second' :default: ret.setTime(ret.getTime() - units*1000); break;
}
return (new Date(ret))
}
s.sqlDate = function(value){
var value = value.toLowerCase()
var splitValue = value.split(' ')
var amount = parseFloat(splitValue[0])
var today = new Date()
var query
if(value.indexOf('min') > -1){
query = s.dateSubtract(today,'minute',amount)
}else if(value.indexOf('day') > -1){
query = s.dateSubtract(today,'day',amount)
}else if(value.indexOf('hour') > -1){
query = s.dateSubtract(today,'hour',amount)
}
return query
}
s.mergeQueryValues = function(query,values){
if(!values){values=[]}
var valuesNotFunction = true;
if(typeof values === 'function'){
var values = [];
valuesNotFunction = false;
}
if(values&&valuesNotFunction){
var splitQuery = query.split('?')
var newQuery = ''
splitQuery.forEach(function(v,n){
newQuery += v
var value = values[n]
if(value){
if(isNaN(value) || value instanceof Date){
newQuery += "'"+value+"'"
}else{
newQuery += value
}
}
})
}else{
newQuery = query
}
return newQuery
}
s.stringToSqlTime = function(value){
newValue = new Date(value.replace('T',' '))
return newValue
}
s.sqlQuery = function(query,values,onMoveOn){
if(!values){values=[]}
if(typeof values === 'function'){
var onMoveOn = values;
var values = [];
}
if(!onMoveOn){onMoveOn=function(){}}
var mergedQuery = s.mergeQueryValues(query,values)
s.debugLog('s.sqlQuery QUERY',mergedQuery)
return s.databaseEngine
.raw(query,values)
.asCallback(function(err,r){
if(err){
console.log('s.sqlQuery QUERY ERRORED',query)
console.log('s.sqlQuery ERROR',err)
}
if(onMoveOn && typeof onMoveOn === 'function'){
switch(databaseOptions.client){
case'sqlite3':
if(!r)r=[]
break;
default:
if(r)r=r[0]
break;
}
onMoveOn(err,r)
}
})
}
if(!config.videosDir)config.videosDir = s.mainDirectory + '/videos/';
if(!config.binDir){config.binDir = s.mainDirectory + '/fileBin/'}
const {
checkCorrectPathEnding,
generateRandomId,
formattedTime,
localToUtc,
} = require('./libs/basic/utils.js')(s.mainDirectory)
const {
sqlDate,
knexQuery,
initiateDatabaseEngine
} = require('./libs/sql/utils.js')(s,config)
var theCronInterval = null
const overlapLocks = {}
const alreadyDeletedRowsWithNoVideosOnStart = {}
const videoDirectory = checkCorrectPathEnding(config.videosDir)
const fileBinDirectory = checkCorrectPathEnding(config.binDir)
s.debugLog = function(arg1,arg2){
if(config.debugLog === true){
if(!arg2)arg2 = ''
console.log(arg1,arg2)
}
}
//containers
var overlapLocks = {}
s.alreadyDeletedRowsWithNoVideosOnStart={};
//functions
s.checkCorrectPathEnding=function(x){
var length=x.length
if(x.charAt(length-1)!=='/'){
x=x+'/'
}
return x.replace('__DIR__',__dirname)
const connectToMainProcess = () => {
const io = require('socket.io-client')('ws://'+config.ip+':'+config.port,{
transports:['websocket']
});
io.on('connect',function(d){
postMessage({
f: 'init',
time: moment()
})
})
io.on('f',function(d){
//command from main process
switch(d.f){
case'start':case'restart':
setIntervalForCron()
break;
case'stop':
clearCronInterval()
break;
}
})
return io
}
s.dir={
videos:s.checkCorrectPathEnding(config.videosDir),
fileBin:s.checkCorrectPathEnding(config.binDir),
addStorage:config.addStorage,
};
s.moment=function(e,x){
if(!e){e=new Date};if(!x){x='YYYY-MM-DDTHH-mm-ss'};
return moment(e).format(x);
const postMessage = (x) => {
x.cronKey = config.cron.key;
return io.emit('cron',x)
}
s.utcToLocal = function(time){
return moment.utc(time).utcOffset(s.utcOffset).format()
const sendToWebSocket = (x,y) => {
//emulate master socket emitter
postMessage({f:'s.tx',data:x,to:y})
}
s.localToUtc = function(time){
return moment(time).utc()
const deleteVideo = (x) => {
postMessage({f:'s.deleteVideo',file:x})
}
s.nameToTime = function(x){x=x.replace('.webm','').replace('.mp4','').split('T'),x[1]=x[1].replace(/-/g,':');x=x.join(' ');return x;}
io = require('socket.io-client')('ws://'+config.ip+':'+config.port,{transports:['websocket']});//connect to master
s.cx = function(x){x.cronKey=config.cron.key;return io.emit('cron',x)}
//emulate master socket emitter
s.tx = function(x,y){s.cx({f:'s.tx',data:x,to:y})}
s.deleteVideo = function(x){s.cx({f:'s.deleteVideo',file:x})}
//Cron Job
s.cx({f:'init',time:moment()})
s.getVideoDirectory = function(e){
const getVideoDirectory = function(e){
if(e.mid&&!e.id){e.id=e.mid};
if(e.details&&(e.details instanceof Object)===false){
try{e.details=JSON.parse(e.details)}catch(err){}
}
if(e.details.dir&&e.details.dir!==''){
return s.checkCorrectPathEnding(e.details.dir)+e.ke+'/'+e.id+'/'
return checkCorrectPathEnding(e.details.dir)+e.ke+'/'+e.id+'/'
}else{
return s.dir.videos+e.ke+'/'+e.id+'/';
return videoDirectory + e.ke + '/' + e.id + '/'
}
}
s.getFileBinDirectory = function(e){
if(e.mid&&!e.id){e.id=e.mid};
return s.dir.fileBin+e.ke+'/'+e.id+'/';
const getFileBinDirectory = function(e){
if(e.mid && !e.id){e.id = e.mid}
return fileBinDirectory + e.ke + '/' + e.id + '/'
}
//filters set by the user in their dashboard
//deleting old videos is part of the filter - config.cron.deleteOld
@ -204,7 +115,7 @@ const checkFilterRules = function(v,callback){
var where = [{
"p1":"end",
"p2":"<=",
"p3":s.sqlDate(v.d.days+" DAY")
"p3": sqlDate(v.d.days+" DAY")
}]
//exclude monitors with their own max days
v.monitorsWithMaxKeepDays.forEach(function(mid){
@ -236,31 +147,26 @@ const checkFilterRules = function(v,callback){
var b = v.d.filters[m];
s.debugLog(b)
if(b.enabled==="1"){
b.ar=[v.ke];
b.sql=[];
b.where.forEach(function(j,k){
if(j.p1==='ke'){j.p3=v.ke}
switch(j.p3_type){
case'function':
b.sql.push(j.p1+' '+j.p2+' '+j.p3)
break;
default:
b.sql.push(j.p1+' '+j.p2+' ?')
b.ar.push(j.p3)
break;
}
const whereQuery = [
['ke','=',v.ke],
['status','!=',"0"],
['details','NOT LIKE','%"archived":"1"%'],
]
b.where.forEach(function(condition){
if(condition.p1 === 'ke'){condition.p3 = v.ke}
whereQuery.push([condition.p1,condition.p2 || '=',condition.p3])
})
b.sql='WHERE ke=? AND status != 0 AND details NOT LIKE \'%"archived":"1"%\' AND ('+b.sql.join(' AND ')+')';
if(b.sort_by&&b.sort_by!==''){
b.sql+=' ORDER BY `'+b.sort_by+'` '+b.sort_by_direction
}
if(b.limit&&b.limit!==''){
b.sql+=' LIMIT '+b.limit
}
s.sqlQuery('SELECT * FROM Videos '+b.sql,b.ar,function(err,r){
if(r&&r[0]){
knexQuery({
action: "select",
columns: "*",
table: "Videos",
where: whereQuery,
orderBy: [b.sort_by,b.sort_by_direction.toLowerCase()],
limit: b.limit
},(err,r) => {
if(r && r[0]){
if(r.length > 0 || config.debugLog === true){
s.cx({f:'filterMatch',msg:r.length+' SQL rows match "'+m+'"',ke:v.ke,time:moment()})
postMessage({f:'filterMatch',msg:r.length+' SQL rows match "'+m+'"',ke:v.ke,time:moment()})
}
b.cx={
f:'filters',
@ -271,9 +177,9 @@ const checkFilterRules = function(v,callback){
id:b.id
};
if(b.archive==="1"){
s.cx({f:'filters',ff:'archive',videos:r,time:moment(),ke:v.ke,id:b.id});
postMessage({f:'filters',ff:'archive',videos:r,time:moment(),ke:v.ke,id:b.id});
}else if(b.delete==="1"){
s.cx({f:'filters',ff:'delete',videos:r,time:moment(),ke:v.ke,id:b.id});
postMessage({f:'filters',ff:'delete',videos:r,time:moment(),ke:v.ke,id:b.id});
}
if(b.email==="1"){
b.cx.ff='email';
@ -281,10 +187,10 @@ const checkFilterRules = function(v,callback){
b.cx.mail=v.mail;
b.cx.execute=b.execute;
b.cx.query=b.sql;
s.cx(b.cx);
postMessage(b.cx);
}
if(b.execute&&b.execute!==""){
s.cx({f:'filters',ff:'execute',execute:b.execute,time:moment()});
postMessage({f:'filters',ff:'execute',execute:b.execute,time:moment()});
}
}
})
@ -305,14 +211,23 @@ const deleteRowsWithNoVideo = function(v,callback){
if(
config.cron.deleteNoVideo===true&&(
config.cron.deleteNoVideoRecursion===true||
(config.cron.deleteNoVideoRecursion===false&&!s.alreadyDeletedRowsWithNoVideosOnStart[v.ke])
(config.cron.deleteNoVideoRecursion===false&&!alreadyDeletedRowsWithNoVideosOnStart[v.ke])
)
){
s.alreadyDeletedRowsWithNoVideosOnStart[v.ke]=true;
es={};
s.sqlQuery('SELECT * FROM Videos WHERE ke=? AND status!=0 AND details NOT LIKE \'%"archived":"1"%\' AND time < ?',[v.ke,s.sqlDate('10 MINUTE')],function(err,evs){
if(evs&&evs[0]){
es.del=[];es.ar=[v.ke];
alreadyDeletedRowsWithNoVideosOnStart[v.ke]=true;
knexQuery({
action: "select",
columns: "*",
table: "Videos",
where: [
['ke','=',v.ke],
['status','!=','0'],
['details','NOT LIKE','%"archived":"1"%'],
['time','<', sqlDate('10 MINUTE')],
]
},(err,evs) => {
if(evs && evs[0]){
const videosToDelete = [];
evs.forEach(function(ev){
var filename
var details
@ -325,20 +240,20 @@ const deleteRowsWithNoVideo = function(v,callback){
details = {}
}
}
var dir = s.getVideoDirectory(ev)
var dir = getVideoDirectory(ev)
if(details.isUTC === true){
filename = s.localToUtc(ev.time).format('YYYY-MM-DDTHH-mm-ss')+'.'+ev.ext
filename = localToUtc(ev.time).format('YYYY-MM-DDTHH-mm-ss')+'.'+ev.ext
}else{
filename = s.moment(ev.time)+'.'+ev.ext
filename = formattedTime(ev.time)+'.'+ev.ext
}
fileExists = fs.existsSync(dir+filename)
if(fileExists !== true){
s.deleteVideo(ev)
s.tx({f:'video_delete',filename:filename+'.'+ev.ext,mid:ev.mid,ke:ev.ke,time:ev.time,end:s.moment(new Date,'YYYY-MM-DD HH:mm:ss')},'GRP_'+ev.ke);
deleteVideo(ev)
sendToWebSocket({f:'video_delete',filename:filename+'.'+ev.ext,mid:ev.mid,ke:ev.ke,time:ev.time,end: formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')},'GRP_'+ev.ke);
}
});
if(es.del.length>0 || config.debugLog === true){
s.cx({f:'deleteNoVideo',msg:es.del.length+' SQL rows with no file deleted',ke:v.ke,time:moment()})
if(videosToDelete.length>0 || config.debugLog === true){
postMessage({f:'deleteNoVideo',msg:videosToDelete.length+' SQL rows with no file deleted',ke:v.ke,time:moment()})
}
}
setTimeout(function(){
@ -353,11 +268,18 @@ const deleteRowsWithNoVideo = function(v,callback){
const deleteOldLogs = function(v,callback){
if(!v.d.log_days||v.d.log_days==''){v.d.log_days=10}else{v.d.log_days=parseFloat(v.d.log_days)};
if(config.cron.deleteLogs===true&&v.d.log_days!==0){
s.sqlQuery("DELETE FROM Logs WHERE ke=? AND `time` < ?",[v.ke,s.sqlDate(v.d.log_days+' DAY')],function(err,rrr){
knexQuery({
action: "delete",
table: "Logs",
where: [
['ke','=',v.ke],
['time','<', sqlDate(v.d.log_days+' DAY')],
]
},(err,rrr) => {
callback()
if(err)return console.error(err);
if(rrr.affectedRows && rrr.affectedRows.length>0 || config.debugLog === true){
s.cx({f:'deleteLogs',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.log_days+' days deleted',ke:v.ke,time:moment()})
postMessage({f:'deleteLogs',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.log_days+' days deleted',ke:v.ke,time:moment()})
}
})
}else{
@ -368,11 +290,40 @@ const deleteOldLogs = function(v,callback){
const deleteOldEvents = function(v,callback){
if(!v.d.event_days||v.d.event_days==''){v.d.event_days=10}else{v.d.event_days=parseFloat(v.d.event_days)};
if(config.cron.deleteEvents===true&&v.d.event_days!==0){
s.sqlQuery("DELETE FROM Events WHERE ke=? AND `time` < ?",[v.ke,s.sqlDate(v.d.event_days+' DAY')],function(err,rrr){
knexQuery({
action: "delete",
table: "Events",
where: [
['ke','=',v.ke],
['time','<', sqlDate(v.d.event_days+' DAY')],
]
},(err,rrr) => {
callback()
if(err)return console.error(err);
if(rrr.affectedRows && rrr.affectedRows.length>0 || config.debugLog === true){
s.cx({f:'deleteEvents',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.event_days+' days deleted',ke:v.ke,time:moment()})
if(rrr.affectedRows && rrr.affectedRows.length > 0 || config.debugLog === true){
postMessage({f:'deleteEvents',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.event_days+' days deleted',ke:v.ke,time:moment()})
}
})
}else{
callback()
}
}
//event counts
const deleteOldEventCounts = function(v,callback){
if(!v.d.event_days||v.d.event_days==''){v.d.event_days=10}else{v.d.event_days=parseFloat(v.d.event_days)};
if(config.cron.deleteEvents===true&&v.d.event_days!==0){
knexQuery({
action: "delete",
table: "Events Counts",
where: [
['ke','=',v.ke],
['time','<', sqlDate(v.d.event_days+' DAY')],
]
},(err,rrr) => {
callback()
if(err && err.code !== 'ER_NO_SUCH_TABLE')return console.error(err);
if(rrr.affectedRows && rrr.affectedRows.length > 0 || config.debugLog === true){
postMessage({f:'deleteEvents',msg:(rrr.affectedRows || 0)+' SQL rows older than '+v.d.event_days+' days deleted',ke:v.ke,time:moment()})
}
})
}else{
@ -384,20 +335,35 @@ const deleteOldFileBins = function(v,callback){
if(!v.d.fileBin_days||v.d.fileBin_days==''){v.d.fileBin_days=10}else{v.d.fileBin_days=parseFloat(v.d.fileBin_days)};
if(config.cron.deleteFileBins===true&&v.d.fileBin_days!==0){
var fileBinQuery = " FROM Files WHERE ke=? AND `time` < ?";
s.sqlQuery("SELECT *"+fileBinQuery,[v.ke,s.sqlDate(v.d.fileBin_days+' DAY')],function(err,files){
knexQuery({
action: "select",
columns: "*",
table: "Files",
where: [
['ke','=',v.ke],
['time','<', sqlDate(v.d.fileBin_days+' DAY')],
]
},(err,files) => {
if(files&&files[0]){
//delete the files
files.forEach(function(file){
fs.unlink(s.getFileBinDirectory(file)+file.name,function(err){
fs.unlink(getFileBinDirectory(file) + file.name,function(err){
// if(err)console.error(err)
})
})
//delete the database rows
s.sqlQuery("DELETE"+fileBinQuery,[v.ke,v.d.fileBin_days],function(err,rrr){
knexQuery({
action: "delete",
table: "Files",
where: [
['ke','=',v.ke],
['time','<', sqlDate(v.d.fileBin_days+' DAY')],
]
},(err,rrr) => {
callback()
if(err)return console.error(err);
if(rrr.affectedRows && rrr.affectedRows.length>0 || config.debugLog === true){
s.cx({f:'deleteFileBins',msg:(rrr.affectedRows || 0)+' files older than '+v.d.fileBin_days+' days deleted',ke:v.ke,time:moment()})
postMessage({f:'deleteFileBins',msg:(rrr.affectedRows || 0)+' files older than '+v.d.fileBin_days+' days deleted',ke:v.ke,time:moment()})
}
})
}else{
@ -424,8 +390,8 @@ const processUser = function(number,rows){
return
}
s.debugLog(v)
if(!s.alreadyDeletedRowsWithNoVideosOnStart[v.ke]){
s.alreadyDeletedRowsWithNoVideosOnStart[v.ke]=false;
if(!alreadyDeletedRowsWithNoVideosOnStart[v.ke]){
alreadyDeletedRowsWithNoVideosOnStart[v.ke]=false;
}
if(!overlapLocks[v.ke]){
// set overlap lock
@ -436,7 +402,14 @@ const processUser = function(number,rows){
if(!v.d.size||v.d.size==''){v.d.size=10000}else{v.d.size=parseFloat(v.d.size)};
//days to keep videos
if(!v.d.days||v.d.days==''){v.d.days=5}else{v.d.days=parseFloat(v.d.days)};
s.sqlQuery('SELECT * FROM Monitors WHERE ke=?', [v.ke], function(err,rr) {
knexQuery({
action: "select",
columns: "*",
table: "Monitors",
where: [
['ke','=',v.ke],
]
},(err,rr) => {
if(!v.d.filters||v.d.filters==''){
v.d.filters={};
}
@ -463,7 +436,7 @@ const processUser = function(number,rows){
},{
"p1":"end",
"p2":"<",
"p3":s.sqlDate(b.details.max_keep_days+" DAY")
"p3": sqlDate(b.details.max_keep_days+" DAY")
}]
};
}
@ -474,14 +447,17 @@ const processUser = function(number,rows){
s.debugLog('--- deleteOldFileBins Complete')
deleteOldEvents(v,function(){
s.debugLog('--- deleteOldEvents Complete')
checkFilterRules(v,function(){
s.debugLog('--- checkFilterRules Complete')
deleteRowsWithNoVideo(v,function(){
s.debugLog('--- deleteRowsWithNoVideo Complete')
checkForOrphanedFiles(v,function(){
//done user, unlock current, and do next
overlapLocks[v.ke]=false;
processUser(number+1,rows)
deleteOldEventCounts(v,function(){
s.debugLog('--- deleteOldEventCounts Complete')
checkFilterRules(v,function(){
s.debugLog('--- checkFilterRules Complete')
deleteRowsWithNoVideo(v,function(){
s.debugLog('--- deleteRowsWithNoVideo Complete')
checkForOrphanedFiles(v,function(){
//done user, unlock current, and do next
overlapLocks[v.ke]=false;
processUser(number+1,rows)
})
})
})
})
@ -494,7 +470,6 @@ const processUser = function(number,rows){
}
}
//recursive function
var theCronInterval = null
const setIntervalForCron = function(){
clearCronInterval()
theCronInterval = setInterval(doCronJobs,parseFloat(config.cron.interval)*60000*60)
@ -503,8 +478,18 @@ const clearCronInterval = function(){
clearInterval(theCronInterval)
}
const doCronJobs = function(){
s.cx({f:'start',time:moment()})
s.sqlQuery('SELECT ke,uid,details,mail FROM Users WHERE details NOT LIKE \'%"sub"%\'', function(err,rows) {
postMessage({
f: 'start',
time: moment()
})
knexQuery({
action: "select",
columns: "ke,uid,details,mail",
table: "Users",
where: [
['details','NOT LIKE','%"sub"%'],
]
},(err,rows) => {
if(err){
console.error(err)
}
@ -513,17 +498,8 @@ const doCronJobs = function(){
}
})
}
initiateDatabaseEngine()
const io = connectToMainProcess()
setIntervalForCron()
doCronJobs()
//socket commander
io.on('f',function(d){
switch(d.f){
case'start':case'restart':
setIntervalForCron()
break;
case'stop':
clearCronInterval()
break;
}
})
console.log('Shinobi : cron.js started')

93
libs/basic/utils.js Normal file
View File

@ -0,0 +1,93 @@
const fs = require('fs');
const path = require('path');
const moment = require('moment');
module.exports = (processCwd) => {
const parseJSON = (string) => {
var parsed
try{
parsed = JSON.parse(string)
}catch(err){
}
if(!parsed)parsed = string
return parsed
}
const stringJSON = (json) => {
try{
if(json instanceof Object){
json = JSON.stringify(json)
}
}catch(err){
}
return json
}
const stringContains = (find,string,toLowerCase) => {
var newString = string + ''
if(toLowerCase)newString = newString.toLowerCase()
return newString.indexOf(find) > -1
}
const checkCorrectPathEnding = (x) => {
var length=x.length
if(x.charAt(length-1)!=='/'){
x=x+'/'
}
return x.replace('__DIR__',processCwd)
}
const mergeDeep = function(...objects) {
const isObject = obj => obj && typeof obj === 'object';
return objects.reduce((prev, obj) => {
Object.keys(obj).forEach(key => {
const pVal = prev[key];
const oVal = obj[key];
if (Array.isArray(pVal) && Array.isArray(oVal)) {
prev[key] = pVal.concat(...oVal);
}
else if (isObject(pVal) && isObject(oVal)) {
prev[key] = mergeDeep(pVal, oVal);
}
else {
prev[key] = oVal;
}
});
return prev;
}, {});
}
const nameToTime = (x) => {
x = x.split('.')[0].split('T')
if(x[1])x[1] = x[1].replace(/-/g,':')
x = x.join(' ')
return x
}
const generateRandomId = (x) => {
if(!x){x=10};var t = "";var p = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for( var i=0; i < x; i++ )
t += p.charAt(Math.floor(Math.random() * p.length));
return t;
}
const utcToLocal = (time) => {
return moment.utc(time).utcOffset(s.utcOffset).format()
}
const localToUtc = (time) => {
return moment(time).utc()
}
const formattedTime = (e,x) => {
if(!e){e=new Date};if(!x){x='YYYY-MM-DDTHH-mm-ss'};
return moment(e).format(x);
}
return {
parseJSON: parseJSON,
stringJSON: stringJSON,
stringContains: stringContains,
checkCorrectPathEnding: checkCorrectPathEnding,
nameToTime: nameToTime,
mergeDeep: mergeDeep,
generateRandomId: generateRandomId,
utcToLocal: utcToLocal,
localToUtc: localToUtc,
formattedTime: formattedTime,
}
}

256
libs/sql/utils.js Normal file
View File

@ -0,0 +1,256 @@
var knex = require('knex');
module.exports = (s,config,databaseOptions) => {
var databaseOptions = {
client: config.databaseType,
connection: config.db,
}
if(databaseOptions.client.indexOf('sqlite')>-1){
databaseOptions.client = 'sqlite3';
databaseOptions.useNullAsDefault = true;
}
if(databaseOptions.client === 'sqlite3' && databaseOptions.connection.filename === undefined){
databaseOptions.connection.filename = __dirname+"/shinobi.sqlite"
}
const dateSubtract = function(date, interval, units){
var ret = date
var checkRollover = function() { if(ret.getDate() != date.getDate()) ret.setDate(0);};
switch(interval.toLowerCase()) {
case 'year' : ret.setFullYear(ret.getFullYear() - units); checkRollover(); break;
case 'quarter': ret.setMonth(ret.getMonth() - 3*units); checkRollover(); break;
case 'month' : ret.setMonth(ret.getMonth() - units); checkRollover(); break;
case 'week' : ret.setDate(ret.getDate() - 7*units); break;
case 'day' : ret.setDate(ret.getDate() - units); break;
case 'hour' : ret.setTime(ret.getTime() - units*3600000); break;
case 'minute' : ret.setTime(ret.getTime() - units*60000); break;
case 'second' :default: ret.setTime(ret.getTime() - units*1000); break;
}
return (new Date(ret))
}
const sqlDate = function(value){
var value = value.toLowerCase()
var splitValue = value.split(' ')
var amount = parseFloat(splitValue[0])
var today = new Date()
var query
if(value.indexOf('min') > -1){
query = dateSubtract(today,'minute',amount)
}else if(value.indexOf('day') > -1){
query = dateSubtract(today,'day',amount)
}else if(value.indexOf('hour') > -1){
query = dateSubtract(today,'hour',amount)
}
return query
}
const mergeQueryValues = function(query,values){
if(!values){values=[]}
var valuesNotFunction = true;
if(typeof values === 'function'){
var values = [];
valuesNotFunction = false;
}
if(values&&valuesNotFunction){
var splitQuery = query.split('?')
var newQuery = ''
splitQuery.forEach(function(v,n){
newQuery += v
var value = values[n]
if(value){
if(isNaN(value) || value instanceof Date){
newQuery += "'"+value+"'"
}else{
newQuery += value
}
}
})
}else{
newQuery = query
}
return newQuery
}
const stringToSqlTime = function(value){
newValue = new Date(value.replace('T',' '))
return newValue
}
const sqlQuery = function(query,values,onMoveOn){
if(!values){values=[]}
if(typeof values === 'function'){
var onMoveOn = values;
var values = [];
}
if(!onMoveOn){onMoveOn=function(){}}
var mergedQuery = mergeQueryValues(query,values)
s.debugLog('s.sqlQuery QUERY',mergedQuery)
return s.databaseEngine
.raw(query,values)
.asCallback(function(err,r){
if(err){
console.log('s.sqlQuery QUERY ERRORED',query)
console.log('s.sqlQuery ERROR',err)
}
if(onMoveOn && typeof onMoveOn === 'function'){
switch(databaseOptions.client){
case'sqlite3':
if(!r)r=[]
break;
default:
if(r)r=r[0]
break;
}
onMoveOn(err,r)
}
})
}
const cleanSqlWhereObject = (where) => {
const newWhere = {}
Object.keys(where).forEach((key) => {
if(key !== '__separator'){
const value = where[key]
newWhere[key] = value
}
})
return newWhere
}
const processSimpleWhereCondition = (dbQuery,where,didOne) => {
var whereIsArray = where instanceof Array;
if(where[0] === 'or' || where.__separator === 'or'){
if(whereIsArray){
where.shift()
dbQuery.orWhere(...where)
}else{
where = cleanSqlWhereObject(where)
dbQuery.orWhere(where)
}
}else if(!didOne){
didOne = true
whereIsArray ? dbQuery.where(...where) : dbQuery.where(where)
}else{
whereIsArray ? dbQuery.andWhere(...where) : dbQuery.andWhere(where)
}
}
const processWhereCondition = (dbQuery,where,didOne) => {
var whereIsArray = where instanceof Array;
if(!where[0])return;
if(where[0] && where[0] instanceof Array){
dbQuery.where(function() {
var _this = this
var didOneInsideGroup = false
where.forEach((whereInsideGroup) => {
processWhereCondition(_this,whereInsideGroup,didOneInsideGroup)
})
})
}else if(where[0] && where[0] instanceof Object){
dbQuery.where(function() {
var _this = this
var didOneInsideGroup = false
where.forEach((whereInsideGroup) => {
processSimpleWhereCondition(_this,whereInsideGroup,didOneInsideGroup)
})
})
}else{
processSimpleWhereCondition(dbQuery,where,didOne)
}
}
const knexError = (dbQuery,options,err) => {
console.error('knexError----------------------------------- START')
if(config.debugLogVerbose && config.debugLog === true){
s.debugLog('s.knexQuery QUERY',JSON.stringify(options,null,3))
s.debugLog('STACK TRACE, NOT AN ',new Error())
}
console.error(err)
console.error(dbQuery.toString())
console.error('knexError----------------------------------- END')
}
const knexQuery = (options,callback) => {
try{
if(!s.databaseEngine)return// console.log('Database Not Set');
// options = {
// action: "",
// columns: "",
// table: ""
// }
var dbQuery
switch(options.action){
case'select':
options.columns = options.columns.indexOf(',') === -1 ? [options.columns] : options.columns.split(',');
dbQuery = s.databaseEngine.select(...options.columns).from(options.table)
break;
case'count':
options.columns = options.columns.indexOf(',') === -1 ? [options.columns] : options.columns.split(',');
dbQuery = s.databaseEngine(options.table)
dbQuery.count(options.columns)
break;
case'update':
dbQuery = s.databaseEngine(options.table).update(options.update)
break;
case'delete':
dbQuery = s.databaseEngine(options.table)
break;
case'insert':
dbQuery = s.databaseEngine(options.table).insert(options.insert)
break;
}
if(options.where instanceof Array){
var didOne = false;
options.where.forEach((where) => {
processWhereCondition(dbQuery,where,didOne)
})
}else if(options.where instanceof Object){
dbQuery.where(options.where)
}
if(options.action === 'delete'){
dbQuery.del()
}
if(options.orderBy){
dbQuery.orderBy(...options.orderBy)
}
if(options.groupBy){
dbQuery.groupBy(options.groupBy)
}
if(options.limit){
if(`${options.limit}`.indexOf(',') === -1){
dbQuery.limit(options.limit)
}else{
const limitParts = `${options.limit}`.split(',')
dbQuery.limit(limitParts[1]).offset(limitParts[0])
}
}
if(config.debugLog === true){
console.log(dbQuery.toString())
}
if(callback || options.update || options.insert || options.action === 'delete'){
dbQuery.asCallback(function(err,r) {
if(err){
knexError(dbQuery,options,err)
}
if(callback)callback(err,r)
if(config.debugLogVerbose && config.debugLog === true){
s.debugLog('s.knexQuery QUERY',JSON.stringify(options,null,3))
s.debugLog('s.knexQuery RESPONSE',JSON.stringify(r,null,3))
s.debugLog('STACK TRACE, NOT AN ',new Error())
}
})
}
return dbQuery
}catch(err){
if(callback)callback(err,[])
knexError(dbQuery,options,err)
}
}
const initiateDatabaseEngine = () => {
s.databaseEngine = knex(databaseOptions)
return s.databaseEngine
}
return {
dateSubtract: dateSubtract,
sqlDate: sqlDate,
mergeQueryValues: mergeQueryValues,
stringToSqlTime: stringToSqlTime,
sqlQuery: sqlQuery,
cleanSqlWhereObject: cleanSqlWhereObject,
processSimpleWhereCondition: processSimpleWhereCondition,
processWhereCondition: processWhereCondition,
knexError: knexError,
knexQuery: knexQuery,
initiateDatabaseEngine: initiateDatabaseEngine
}
}