ShinobiSystems
/
Shinobi
mirror of https://gitlab.com/Shinobi-Systems/Shinobi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update some delete queries

const-conversion
Moe 2020-08-07 15:12:45 -07:00
parent 52a84f4501
commit 4be1659873
10 changed files with 185 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libs/scheduler.js
View File

@ -317,10 +317,10 @@ module.exports = function(s,config,lang,app,io){
s.knexQuery({
action: "delete",
table: "Schedules",
where: [
['ke','=',req.params.ke],
['name','=',req.params.name],
]
where : {
ke: req.params.ke,
name: req.params.name,
}
},function(err){
if(!err){
endData.msg = lang["Deleted Schedule Configuration"]

6
libs/socketio.js
View File

@ -752,9 +752,9 @@ module.exports = function(s,config,lang,io){
s.knexQuery({
action: "delete",
table: "Logs",
where: [
['ke','=',d.ke],
]
where: {
ke: d.ke,
}
})
break;
}

68
libs/sql.js
View File

@ -61,28 +61,25 @@ module.exports = function(s,config){
.raw(data.query,data.values)
.asCallback(callback)
}, 4);
const processWhereCondition = (dbQuery,where,didOne) => {
const cleanSqlWhereObject = (where) => {
const newWhere = {}
Object.keys(where).forEach((key) => {
if(key !== '__separator'){
const value = where[key]
newWhere[key] = value
}
})
return newWhere
}
const processSimpleWhereCondition = (dbQuery,where,didOne) => {
var whereIsArray = where instanceof Array;
if(!where[0])return;
if(where[0] && where[0] instanceof Array){
// didOne = true
dbQuery.where(function() {
var _this = this
var didOneInsideGroup = false
where.forEach((whereInsideGroup) => {
processWhereCondition(_this,whereInsideGroup,didOneInsideGroup)
})
})
}else if(where.length === 4){
const separator = where[0] + ''
where.shift()
switch(separator){
case'and':
whereIsArray ? dbQuery.andWhere(...where) : dbQuery.andWhere(where)
break;
case'or':
whereIsArray ? dbQuery.orWhere(...where) : dbQuery.orWhere(where)
break;
if(where[0] === 'or' || where.__separator === 'or'){
if(whereIsArray){
where.shift()
dbQuery.orWhere(...where)
}else{
where = cleanSqlWhereObject(where)
dbQuery.orWhere(where)
}
}else if(!didOne){
didOne = true
@ -90,7 +87,29 @@ module.exports = function(s,config){
}else{
whereIsArray ? dbQuery.andWhere(...where) : dbQuery.andWhere(where)
}
}
const processWhereCondition = (dbQuery,where,didOne) => {
var whereIsArray = where instanceof Array;
if(!where[0])return;
if(where[0] && where[0] instanceof Array){
dbQuery.where(function() {
var _this = this
var didOneInsideGroup = false
where.forEach((whereInsideGroup) => {
processWhereCondition(_this,whereInsideGroup,didOneInsideGroup)
})
})
}else if(where[0] && where[0] instanceof Object){
dbQuery.where(function() {
var _this = this
var didOneInsideGroup = false
where.forEach((whereInsideGroup) => {
processSimpleWhereCondition(_this,whereInsideGroup,didOneInsideGroup)
})
})
}else{
processSimpleWhereCondition(dbQuery,where,didOne)
}
}
const knexError = (dbQuery,options,err) => {
console.error('knexError----------------------------------- START')
@ -125,7 +144,7 @@ module.exports = function(s,config){
dbQuery = s.databaseEngine(options.table).update(options.update)
break;
case'delete':
dbQuery = s.databaseEngine(options.table).del()
dbQuery = s.databaseEngine(options.table)
break;
case'insert':
dbQuery = s.databaseEngine(options.table).insert(options.insert)
@ -137,6 +156,9 @@ module.exports = function(s,config){
processWhereCondition(dbQuery,where,didOne)
})
}
if(options.action === 'delete'){
dbQuery.del()
}
if(options.orderBy){
dbQuery.orderBy(...options.orderBy)
}

4
libs/uploaders/amazonS3.js
View File

@ -117,7 +117,7 @@ module.exports = function(s,config,lang){
href: data.Location
}
})
s.setCloudDiskUsedForGroup(e,{
s.setCloudDiskUsedForGroup(e.ke,{
amount: k.filesizeMB,
storageType: 's3'
})
@ -160,7 +160,7 @@ module.exports = function(s,config,lang){
href: data.Location
}
})
s.setCloudDiskUsedForGroup(e,{
s.setCloudDiskUsedForGroup(e.ke,{
amount : s.kilobyteToMegabyte(queryInfo.size),
storageType : 's3'
},'timelapseFrames')

2
libs/uploaders/backblazeB2.js
View File

@ -148,7 +148,7 @@ module.exports = function(s,config,lang){
href: backblazeDownloadUrl
}
})
s.setCloudDiskUsedForGroup(e,{
s.setCloudDiskUsedForGroup(e.ke,{
amount : k.filesizeMB,
storageType : 'b2'
})

4
libs/uploaders/googleDrive.js
View File

@ -174,7 +174,7 @@ module.exports = (s,config,lang,app,io) => {
href: ''
}
})
s.setCloudDiskUsedForGroup(e,{
s.setCloudDiskUsedForGroup(e.ke,{
amount : k.filesizeMB,
storageType : 'googd'
})
@ -226,7 +226,7 @@ module.exports = (s,config,lang,app,io) => {
href: ''
}
})
s.setCloudDiskUsedForGroup(e,{
s.setCloudDiskUsedForGroup(e.ke,{
amount : s.kilobyteToMegabyte(queryInfo.size),
storageType : 'googd'
},'timelapseFrames')

4
libs/uploaders/s3based.js
View File

@ -134,7 +134,7 @@ module.exports = function(s,config,lang){
href: cloudLink
}
})
s.setCloudDiskUsedForGroup(e,{
s.setCloudDiskUsedForGroup(e.ke,{
amount : k.filesizeMB,
storageType : 'whcs'
})
@ -177,7 +177,7 @@ module.exports = function(s,config,lang){
href: data.Location
}
})
s.setCloudDiskUsedForGroup(e,{
s.setCloudDiskUsedForGroup(e.ke,{
amount : s.kilobyteToMegabyte(queryInfo.size),
storageType : 'whcs'
},'timelapseFrames')

2
libs/uploaders/webdav.js
View File

@ -98,7 +98,7 @@ module.exports = function(s,config,lang){
href: webdavRemoteUrl
}
})
s.setCloudDiskUsedForGroup(e,{
s.setCloudDiskUsedForGroup(e.ke,{
amount: k.filesizeMB,
storageType: 'webdav'
})

142
libs/user.js
View File

@ -12,6 +12,8 @@ module.exports = function(s,config,lang){
deleteMainVideos,
deleteTimelapseFrames,
deleteFileBinFiles,
deleteCloudVideos,
deleteCloudTimelapseFrames,
} = require("./user/utils.js")(s,config,lang);
let purgeDiskGroup = () => {}
const runQuery = async.queue(function(groupKey, callback) {
@ -64,10 +66,10 @@ module.exports = function(s,config,lang){
s.group[e.ke].diskUsedEmitter.emit('purgeCloud',storageType,storagePoint)
}
}
s.setCloudDiskUsedForGroup = function(e,usage,storagePoint){
s.setCloudDiskUsedForGroup = function(groupKey,usage,storagePoint){
//`usage` will be used as the value to add or substract
if(s.group[e.ke].diskUsedEmitter){
s.group[e.ke].diskUsedEmitter.emit('setCloud',usage,storagePoint)
if(s.group[groupKey].diskUsedEmitter){
s.group[groupKey].diskUsedEmitter.emit('setCloud',usage,storagePoint)
}
}
s.sendDiskUsedAmountToClients = function(groupKey){
@ -184,137 +186,15 @@ module.exports = function(s,config,lang){
break;
}
})
s.group[e.ke].diskUsedEmitter.on('purgeCloud',function(storageType,storagePoint){
if(config.cron.deleteOverMax === true){
var cloudDisk = s.group[e.ke].cloudDiskUse[storageType]
//set queue processor
var finish=function(){
// s.sendDiskUsedAmountToClients(e.ke)
}
var deleteVideos = function(){
//run purge command
if(cloudDisk.sizeLimitCheck && cloudDisk.usedSpace > (cloudDisk.sizeLimit*config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Cloud Videos",
where: [
['status','!=','0'],
['ke','=',e.ke],
['details','LIKE',`%"type":"${storageType}"%`],
],
orderBy: ['time','asc'],
limit: 2
},function(err,videos) {
if(!videos)return console.log(err)
var whereQuery = [
['ke','=',e.ke],
[]
]
var didOne = false
videos.forEach(function(video){
video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext
var whereGroup
if(didOne){
whereGroup = [
['or','mid','=',video.mid],
['time','=',video.time]
]
}else{
didOne = true
whereGroup = [
['mid','=',video.mid],
['time','=',video.time]
]
}
whereQuery[1].push(whereGroup)
s.setCloudDiskUsedForGroup(e,{
amount : -(video.size/1048576),
storageType : storageType
})
s.deleteVideoFromCloudExtensionsRunner(e,storageType,video)
})
if(whereQuery[1].length > 0){
s.knexQuery({
action: "delete",
table: "Cloud Videos",
where: whereQuery
},() => {
deleteVideos()
})
}else{
finish()
}
})
}else{
finish()
}
}
var deleteTimelapseFrames = function(callback){
reRunCheck = function(){
return deleteTimelapseFrames(callback)
}
//run purge command
if(cloudDisk.usedSpaceTimelapseFrames > (cloudDisk.sizeLimit * (s.group[e.ke].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Cloud Timelapse Frames",
where: [
['ke','=',e.ke],
['details','NOT LIKE',`%"archived":"1"%`],
],
orderBy: ['time','asc'],
limit: 3
},(err,frames) => {
if(!frames)return console.log(err)
var whereQuery = [
['ke','=',e.ke],
[]
]
frames.forEach(function(frame){
frame.dir = s.getVideoDirectory(frame) + s.formattedTime(frame.time) + '.' + frame.ext
var whereGroup
if(didOne){
whereGroup = [
['or','mid','=',frame.mid],
['time','=',frame.time]
]
}else{
didOne = true
whereGroup = [
['mid','=',frame.mid],
['time','=',frame.time]
]
}
whereQuery[1].push(whereGroup)
s.setCloudDiskUsedForGroup(e,{
amount : -(frame.size/1048576),
storageType : storageType
})
s.deleteVideoFromCloudExtensionsRunner(e,storageType,frame)
})
s.knexQuery({
action: "delete",
table: "Cloud Timelapse Frames",
where: whereQuery
},() => {
deleteTimelapseFrames(callback)
})
})
}else{
callback()
}
}
deleteVideos(function(){
deleteTimelapseFrames(function(){
if(config.cron.deleteOverMax === true){
s.group[e.ke].diskUsedEmitter.on('purgeCloud',function(storageType,storagePoint){
deleteCloudVideos(storageType,storagePoint,function(){
deleteCloudTimelapseFrames(storageType,storagePoint,function(){
})
})
}else{
// s.sendDiskUsedAmountToClients(e.ke)
}
})
})
}
//s.setDiskUsedForGroup
s.group[e.ke].diskUsedEmitter.on('set',function(currentChange,storageType){
//validate current values

121
libs/user/utils.js
View File

@ -15,13 +15,12 @@ module.exports = (s,config,lang) => {
var didOne = false
videos.forEach(function(video){
video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext
if(didOne){
whereGroup.push(['or','mid','=',video.mid])
}else{
didOne = true
whereGroup.push(['mid','=',video.mid])
const queryGroup = {
mid: video.mid,
time: video.time,
}
whereGroup.push(['time','=',video.time])
if(whereGroup.length > 0)queryGroup.__separator = 'or'
whereGroup.push(queryGroup)
fs.chmod(video.dir,0o777,function(err){
fs.unlink(video.dir,function(err){
++completedCheck
@ -32,7 +31,7 @@ module.exports = (s,config,lang) => {
}
})
}
const whereGroupLength = whereGroup.length / 2
const whereGroupLength = whereGroup.length
if(whereGroupLength > 0 && whereGroupLength === completedCheck){
whereQuery[1] = whereGroup
s.knexQuery({
@ -357,6 +356,112 @@ module.exports = (s,config,lang) => {
callback()
}
}
const deleteCloudVideos = function(groupKey,storageType,storagePoint,callback){
const whereGroup = []
const cloudDisk = s.group[groupKey].cloudDiskUse[storageType]
//run purge command
if(cloudDisk.sizeLimitCheck && cloudDisk.usedSpace > (cloudDisk.sizeLimit * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Cloud Videos",
where: [
['status','!=','0'],
['ke','=',groupKey],
['details','LIKE',`%"type":"${storageType}"%`],
],
orderBy: ['time','asc'],
limit: 2
},function(err,videos) {
if(!videos)return console.log(err)
var whereQuery = [
['ke','=',groupKey],
]
var didOne = false
videos.forEach(function(video){
video.dir = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext
const queryGroup = {
mid: video.mid,
time: video.time,
}
if(whereGroup.length > 0)queryGroup.__separator = 'or'
whereGroup.push(queryGroup)
s.setCloudDiskUsedForGroup(e.ke,{
amount : -(video.size/1048576),
storageType : storageType
})
s.deleteVideoFromCloudExtensionsRunner(e,storageType,video)
})
const whereGroupLength = whereGroup.length
if(whereGroupLength > 0){
whereQuery[1] = whereGroup
s.knexQuery({
action: "delete",
table: "Cloud Videos",
where: whereQuery
},() => {
deleteCloudVideos(groupKey,storageType,storagePoint,callback)
})
}else{
callback()
}
})
}else{
callback()
}
}
const deleteCloudTimelapseFrames = function(groupKey,storageType,storagePoint,callback){
const whereGroup = []
var cloudDisk = s.group[e.ke].cloudDiskUse[storageType]
//run purge command
if(cloudDisk.usedSpaceTimelapseFrames > (cloudDisk.sizeLimit * (s.group[e.ke].sizeLimitTimelapseFramesPercent / 100) * config.cron.deleteOverMaxOffset)){
s.knexQuery({
action: "select",
columns: "*",
table: "Cloud Timelapse Frames",
where: [
['ke','=',e.ke],
['details','NOT LIKE',`%"archived":"1"%`],
],
orderBy: ['time','asc'],
limit: 3
},(err,frames) => {
if(!frames)return console.log(err)
var whereQuery = [
['ke','=',e.ke],
]
frames.forEach(function(frame){
frame.dir = s.getVideoDirectory(frame) + s.formattedTime(frame.time) + '.' + frame.ext
const queryGroup = {
mid: frame.mid,
time: frame.time,
}
if(whereGroup.length > 0)queryGroup.__separator = 'or'
whereGroup.push(queryGroup)
s.setCloudDiskUsedForGroup(e.ke,{
amount : -(frame.size/1048576),
storageType : storageType
})
s.deleteVideoFromCloudExtensionsRunner(e,storageType,frame)
})
const whereGroupLength = whereGroup.length
if(whereGroupLength > 0){
whereQuery[1] = whereGroup
s.knexQuery({
action: "delete",
table: "Cloud Timelapse Frames",
where: whereQuery
},() => {
deleteCloudTimelapseFrames(groupKey,storageType,storagePoint,callback)
})
}else{
callback()
}
})
}else{
callback()
}
}
return {
deleteSetOfVideos: deleteSetOfVideos,
deleteSetOfTimelapseFrames: deleteSetOfTimelapseFrames,
@ -365,5 +470,7 @@ module.exports = (s,config,lang) => {
deleteMainVideos: deleteMainVideos,
deleteTimelapseFrames: deleteTimelapseFrames,
deleteFileBinFiles: deleteFileBinFiles,
deleteCloudVideos: deleteCloudVideos,
deleteCloudTimelapseFrames: deleteCloudTimelapseFrames,
}
}