From c589df31e036ffb95c3c29a9d213001431c5fe51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=A7=E9=A5=BC?= Date: Wed, 9 Aug 2023 06:14:24 +0000 Subject: [PATCH 001/112] Update preQueries.js Add logs first . because it has some error when something is done and it will write to logs but table logs is no exist now as Shinobi run at an empty db. --- libs/database/preQueries.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libs/database/preQueries.js b/libs/database/preQueries.js index 0726f896..c2e0beac 100644 --- a/libs/database/preQueries.js +++ b/libs/database/preQueries.js @@ -6,6 +6,16 @@ module.exports = function(s,config){ isMySQL, } = require('./utils.js')(s,config) s.preQueries = async function(){ + await createTable('Logs',[ + isMySQL ? {name: 'utf8', type: 'charset'} : null, + isMySQL ? {name: 'utf8_general_ci', type: 'collate'} : null, + {name: 'ke', length: 50, type: 'string'}, + {name: 'mid', length: 100, type: 'string'}, + {name: 'info', type: 'text'}, + {name: 'time', type: 'timestamp', defaultTo: currentTimestamp()}, + // KEY `logs_index` (`ke`,`mid`,`time`) + {name: ['ke', 'mid', 'time'], type: 'index', length: 'logs_index'}, + ]); await createTable('Users',[ isMySQL ? {name: 'utf8', type: 'charset'} : null, isMySQL ? {name: 'utf8_general_ci', type: 'collate'} : null, @@ -130,16 +140,6 @@ module.exports = function(s,config){ {name: 'size', type: 'bigint'}, {name: 'details', type: 'text'}, ]); - await createTable('Logs',[ - isMySQL ? {name: 'utf8', type: 'charset'} : null, - isMySQL ? {name: 'utf8_general_ci', type: 'collate'} : null, - {name: 'ke', length: 50, type: 'string'}, - {name: 'mid', length: 100, type: 'string'}, - {name: 'info', type: 'text'}, - {name: 'time', type: 'timestamp', defaultTo: currentTimestamp()}, - // KEY `logs_index` (`ke`,`mid`,`time`) - {name: ['ke', 'mid', 'time'], type: 'index', length: 'logs_index'}, - ]); await createTable('Monitors',[ isMySQL ? {name: 'utf8', type: 'charset'} : null, isMySQL ? {name: 'utf8_general_ci', type: 'collate'} : null, From 0d935c17786bec62ab9b3e56fadfa627da25c558 Mon Sep 17 00:00:00 2001 From: Moe Date: Thu, 14 Sep 2023 21:24:23 -0700 Subject: [PATCH 002/112] Fix Auto Port set for RTMP on startup --- definitions/base.js | 3 +-- libs/ffmpeg/possibleWarnings.js | 7 ++++--- libs/ffmpeg/utils.js | 13 ++++++++++++ libs/monitor.js | 13 +++++++++++- libs/monitor/utils.js | 36 ++++++++++++++++++++------------- 5 files changed, 52 insertions(+), 20 deletions(-) diff --git a/definitions/base.js b/definitions/base.js index 80f6498b..136b50b1 100644 --- a/definitions/base.js +++ b/definitions/base.js @@ -748,8 +748,7 @@ module.exports = function(s,config,lang){ "name": "detail=stream_type", "field": lang["Stream Type"], "description": lang["fieldTextStreamType"], - "default": "mp4", - "example": "", + "default": "hls", "selector": "h_st", "fieldType": "select", "attribute": `triggerChange="#add_monitor [detail=stream_vcodec]" triggerChangeIgnore="b64,mjpeg,jpeg,gif"`, diff --git a/libs/ffmpeg/possibleWarnings.js b/libs/ffmpeg/possibleWarnings.js index 8f4097bc..d271743b 100644 --- a/libs/ffmpeg/possibleWarnings.js +++ b/libs/ffmpeg/possibleWarnings.js @@ -1,5 +1,6 @@ module.exports = (monitor,probeResult,config,lang) => { const primaryVideoStream = probeResult.video[0] + const noPortOr554 = !monitor.port || monitor.port === 554 return [ { isTrue: monitor.details.stream_vcodec === 'copy' && primaryVideoStream.codec === 'hevc', @@ -83,7 +84,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && monitor.protocol === 'rtmp' ), title: lang['Automatic Field Fill'], @@ -95,7 +96,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && monitor.protocol === 'http' ), title: lang['Automatic Field Fill'], @@ -107,7 +108,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && (monitor.protocol === 'rtmps' || monitor.protocol === 'https') ), diff --git a/libs/ffmpeg/utils.js b/libs/ffmpeg/utils.js index e20402ec..e678051b 100644 --- a/libs/ffmpeg/utils.js +++ b/libs/ffmpeg/utils.js @@ -367,6 +367,18 @@ Run "npm install ffbinaries" to get this static FFmpeg downloader.` } return response } + async function getWarningChangesForMonitor(monitorConfig){ + const probeResponse = await probeMonitor(monitorConfig,2000,true) + const probeStreams = getStreamInfoFromProbe(probeResponse.result) + const warnings = createWarningsForConfiguration(monitorConfig,probeStreams) + const configPartial = warnings.length > 0 ? buildMonitorConfigPartialFromWarnings(warnings) : {} + return { + configPartial, + warnings, + probeResponse, + probeStreams, + } + } return { ffprobe: runFFprobe, probeMonitor: probeMonitor, @@ -386,5 +398,6 @@ Run "npm install ffbinaries" to get this static FFmpeg downloader.` checkStaticBuilds: checkStaticBuilds, checkVersion: checkVersion, checkHwAccelMethods: checkHwAccelMethods, + getWarningChangesForMonitor, } } diff --git a/libs/monitor.js b/libs/monitor.js index 4a366722..95506de6 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -13,6 +13,8 @@ module.exports = function(s,config,lang){ } = require('./basic/utils.js')(process.cwd(),config) const { splitForFFMPEG, + applyPartialToConfiguration, + getWarningChangesForMonitor, } = require('./ffmpeg/utils.js')(s,config,lang) const { processKill, @@ -551,7 +553,6 @@ module.exports = function(s,config,lang){ return } form.mid = form.mid.replace(/[^\w\s]/gi,'').replace(/ /g,'') - form = s.cleanMonitorObjectForDatabase(form) const selectResponse = await s.knexQueryPromise({ action: "select", columns: "*", @@ -570,6 +571,16 @@ module.exports = function(s,config,lang){ ke: form.ke, mon: form } + // auto correct + const { + configPartial, + warnings, + probeResponse, + probeStreams, + } = await getWarningChangesForMonitor(form) + applyPartialToConfiguration(form,configPartial) + form = s.cleanMonitorObjectForDatabase(form) + // if(monitorExists){ txData.new = false Object.keys(form).forEach(function(v){ diff --git a/libs/monitor/utils.js b/libs/monitor/utils.js index 5db663da..024124e5 100644 --- a/libs/monitor/utils.js +++ b/libs/monitor/utils.js @@ -10,11 +10,8 @@ const SoundDetection = require('shinobi-sound-detection') const streamViewerCountTimeouts = {} module.exports = (s,config,lang) => { const { - probeMonitor, - getStreamInfoFromProbe, applyPartialToConfiguration, - createWarningsForConfiguration, - buildMonitorConfigPartialFromWarnings, + getWarningChangesForMonitor, createPipeArray, splitForFFMPEG, sanitizedFfmpegCommand, @@ -40,6 +37,9 @@ module.exports = (s,config,lang) => { selectNodeForOperation, bindMonitorToChildNode } = require('../childNode/utils.js')(s,config,lang) + const { + asyncSetTimeout, + } = require('../basic/utils.js')(process.cwd(),config) const isMasterNode = ( ( config.childNodes.enabled === true && @@ -769,6 +769,7 @@ module.exports = (s,config,lang) => { code: 4, }); await s.camera('stop',e) + await asyncSetTimeout(2500) if(e.mode !== 'restart')await s.camera(`${e.mode}`,e); } function monitorAddViewer(e,cn){ @@ -898,7 +899,7 @@ module.exports = (s,config,lang) => { status: lang.Restarting, code: 4, }) - await launchMonitorProcesses(monitorConfig) + await monitorRestart(monitorConfig) s.userLog({ ke: groupKey, mid: monitorId, @@ -1353,6 +1354,7 @@ module.exports = (s,config,lang) => { // doFatalErrorCatch(e,d) // },15000) break; + case checkLog(d,'Could not find codec parameters'): case checkLog(d,'No route to host'): activeMonitor.timeoutToRestart = setTimeout(async () => { doFatalErrorCatch(e,d) @@ -1662,17 +1664,18 @@ module.exports = (s,config,lang) => { return; } if(config.probeMonitorOnStart === true){ - const probeResponse = await probeMonitor(monitorConfig,2000,true) - const probeStreams = getStreamInfoFromProbe(probeResponse.result) - activeMonitor.probeResult = probeStreams - const warnings = createWarningsForConfiguration(monitorConfig,probeStreams) - activeMonitor.warnings = warnings + const { + configPartial, + warnings, + probeResponse, + probeStreams, + } = await getWarningChangesForMonitor(monitorConfig) if(warnings.length > 0){ - const configPartial = buildMonitorConfigPartialFromWarnings(warnings) applyPartialToConfiguration(e,configPartial) applyPartialToConfiguration(activeMonitor,configPartial) applyPartialToConfiguration(s.group[groupKey].rawMonitorConfigurations[monitorId],configPartial) } + activeMonitor.warnings = warnings } activeMonitor.isStarted = true if(e.details && e.details.dir && e.details.dir !== ''){ @@ -1694,6 +1697,7 @@ module.exports = (s,config,lang) => { } try{ await launchMonitorProcesses(e) + resetStreamCheck(e) }catch(err){ console.error(err) } @@ -1748,9 +1752,13 @@ module.exports = (s,config,lang) => { } function isGroupBelowMaxMonitorCount(groupKey){ const theGroup = s.group[groupKey]; - const initData = theGroup.init; - const maxCamerasAllowed = parseInt(initData.max_camera) || false; - return (!maxCamerasAllowed || Object.keys(theGroup.activeMonitors).length <= parseInt(maxCamerasAllowed)) + try{ + const initData = theGroup.init; + const maxCamerasAllowed = parseInt(initData.max_camera) || false; + return (!maxCamerasAllowed || Object.keys(theGroup.activeMonitors).length <= parseInt(maxCamerasAllowed)) + }catch(err){ + return true + } } function getStreamDirectory(options){ const streamDir = s.dir.streams + options.ke + '/' + options.mid + '/' From f22d4d180053b24723960455f94eb1448f5cce47 Mon Sep 17 00:00:00 2001 From: Moe Date: Thu, 14 Sep 2023 21:26:20 -0700 Subject: [PATCH 003/112] undo some experiments --- libs/monitor/utils.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libs/monitor/utils.js b/libs/monitor/utils.js index 024124e5..00cd7ddf 100644 --- a/libs/monitor/utils.js +++ b/libs/monitor/utils.js @@ -37,9 +37,6 @@ module.exports = (s,config,lang) => { selectNodeForOperation, bindMonitorToChildNode } = require('../childNode/utils.js')(s,config,lang) - const { - asyncSetTimeout, - } = require('../basic/utils.js')(process.cwd(),config) const isMasterNode = ( ( config.childNodes.enabled === true && @@ -769,7 +766,6 @@ module.exports = (s,config,lang) => { code: 4, }); await s.camera('stop',e) - await asyncSetTimeout(2500) if(e.mode !== 'restart')await s.camera(`${e.mode}`,e); } function monitorAddViewer(e,cn){ @@ -899,7 +895,7 @@ module.exports = (s,config,lang) => { status: lang.Restarting, code: 4, }) - await monitorRestart(monitorConfig) + await launchMonitorProcesses(monitorConfig) s.userLog({ ke: groupKey, mid: monitorId, From dcd69e3c09fd20ad950313870f1349c2a2e9e6e8 Mon Sep 17 00:00:00 2001 From: Moe Date: Sat, 16 Sep 2023 07:53:20 -0700 Subject: [PATCH 004/112] Add "noDefaultRecordingSegmentFormatOptions" bool --- libs/ffmpeg/builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/ffmpeg/builders.js b/libs/ffmpeg/builders.js index 727ee359..faa52386 100644 --- a/libs/ffmpeg/builders.js +++ b/libs/ffmpeg/builders.js @@ -548,7 +548,7 @@ module.exports = (s,config,lang) => { recordingFilters.push(`fps=${videoFps}`) } } - if(videoExtIsMp4){ + if(videoExtIsMp4 && !config.noDefaultRecordingSegmentFormatOptions){ customRecordingFlags.push(`-segment_format_options movflags=faststart`) } if(videoCodec === 'h264_vaapi'){ From cfa9c926db492a31c1fee4a88ac2e694003bf9f1 Mon Sep 17 00:00:00 2001 From: Moe Date: Sun, 17 Sep 2023 07:36:10 -0700 Subject: [PATCH 005/112] fix missing matrices array in getObjectTagsFromMatrices --- libs/events/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/events/utils.js b/libs/events/utils.js index 98e50cdb..22f5bee5 100644 --- a/libs/events/utils.js +++ b/libs/events/utils.js @@ -837,7 +837,7 @@ module.exports = (s,config,lang) => { if(d.details.reason === 'motion'){ return [getTagWithIcon(lang.Motion)] }else{ - const matrices = d.details.matrices + const matrices = d.details.matrices || [] return [...new Set(matrices.map(matrix => getTagWithIcon(matrix.tag)))]; } } From 26713bde3c56f86004d862b41446bda54e368a11 Mon Sep 17 00:00:00 2001 From: Moe Date: Sun, 17 Sep 2023 07:48:57 -0700 Subject: [PATCH 006/112] better patch for getObjectTagsFromMatrices --- libs/events/utils.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/events/utils.js b/libs/events/utils.js index 22f5bee5..f366fb8d 100644 --- a/libs/events/utils.js +++ b/libs/events/utils.js @@ -836,10 +836,11 @@ module.exports = (s,config,lang) => { function getObjectTagsFromMatrices(d){ if(d.details.reason === 'motion'){ return [getTagWithIcon(lang.Motion)] - }else{ - const matrices = d.details.matrices || [] + }else if(d.details.matrices){ + const matrices = d.details.matrices return [...new Set(matrices.map(matrix => getTagWithIcon(matrix.tag)))]; } + return [getTagWithIcon(d.details.reason)] } function getObjectTagNotifyText(d){ const monitorId = d.mid || d.id From 83fbd5f623d48e95a3cd6e790cd36fceddb6b147 Mon Sep 17 00:00:00 2001 From: Moe Date: Mon, 18 Sep 2023 12:01:24 -0700 Subject: [PATCH 007/112] add onCloudVideoUploaded action handler to backblaze --- libs/extenders.js | 1 + libs/uploaders/backblazeB2.js | 52 +++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/libs/extenders.js b/libs/extenders.js index 61b66361..51535c93 100644 --- a/libs/extenders.js +++ b/libs/extenders.js @@ -62,6 +62,7 @@ module.exports = function(s,config){ createExtension(`onEventBasedRecordingComplete`) createExtension(`onEventBasedRecordingStart`) createExtension(`onBeforeInsertCompletedVideo`) + createExtension(`onCloudVideoUploaded`) /////// TIMELAPSE //////// createExtension(`onInsertTimelapseFrame`) } diff --git a/libs/uploaders/backblazeB2.js b/libs/uploaders/backblazeB2.js index c28b0d90..b06294fb 100644 --- a/libs/uploaders/backblazeB2.js +++ b/libs/uploaders/backblazeB2.js @@ -3,18 +3,19 @@ const { Readable } = require('stream'); const B2 = require('backblaze-b2') module.exports = function(s,config,lang){ //Backblaze B2 + var serviceProvider = 'b2' var beforeAccountSaveForBackblazeB2 = function(d){ //d = save event d.formDetails.b2_use_global=d.d.b2_use_global d.formDetails.use_bb_b2=d.d.use_bb_b2 } var cloudDiskUseStartupForBackblazeB2 = function(group,userDetails){ - group.cloudDiskUse['b2'].name = 'Backblaze B2' - group.cloudDiskUse['b2'].sizeLimitCheck = (userDetails.use_bb_b2_size_limit === '1') + group.cloudDiskUse[serviceProvider].name = 'Backblaze B2' + group.cloudDiskUse[serviceProvider].sizeLimitCheck = (userDetails.use_bb_b2_size_limit === '1') if(!userDetails.bb_b2_size_limit || userDetails.bb_b2_size_limit === ''){ - group.cloudDiskUse['b2'].sizeLimit = 10000 + group.cloudDiskUse[serviceProvider].sizeLimit = 10000 }else{ - group.cloudDiskUse['b2'].sizeLimit = parseFloat(userDetails.bb_b2_size_limit) + group.cloudDiskUse[serviceProvider].sizeLimit = parseFloat(userDetails.bb_b2_size_limit) } } var loadBackblazeB2ForUser = function(e){ @@ -83,7 +84,7 @@ module.exports = function(s,config,lang){ }catch(err){ var videoDetails = video.details } - if(video.type !== 'b2'){ + if(video.type !== serviceProvider){ callback() return } @@ -128,30 +129,35 @@ module.exports = function(s,config,lang){ }).then(function(resp){ const uploadResponse = resp.data if(theGroup.init.bb_b2_log === '1' && uploadResponse.fileId){ + const insertDetails = { + bucketId : uploadResponse.bucketId, + fileId : uploadResponse.fileId, + fileName : uploadResponse.fileName + } + const insertQuery = { + mid: e.mid, + ke: e.ke, + time: k.startTime, + status: 1, + type : serviceProvider, + details: insertDetails, + size: k.filesize, + end: k.endTime, + href: '' + } s.knexQuery({ action: "insert", table: "Cloud Videos", - insert: { - mid: e.mid, - ke: e.ke, - time: k.startTime, - status: 1, - type : 'b2', - details: s.s({ - bucketId : uploadResponse.bucketId, - fileId : uploadResponse.fileId, - fileName : uploadResponse.fileName - }), - size: k.filesize, - end: k.endTime, - href: '' - } + insert: Object.assign({},insertQuery,{details: s.s(insertDetails)}) }) s.setCloudDiskUsedForGroup(e.ke,{ amount : k.filesizeMB, - storageType : 'b2' + storageType : serviceProvider + }) + s.purgeCloudDiskForGroup(e,serviceProvider) + s.onCloudVideoUploadedExtensions.forEach((extender) => { + extender(insertQuery) }) - s.purgeCloudDiskForGroup(e,'b2') } }).catch(backblazeErr) }) @@ -183,7 +189,7 @@ module.exports = function(s,config,lang){ } //backblaze b2 s.addCloudUploader({ - name: 'b2', + name: serviceProvider, loadGroupAppExtender: loadBackblazeB2ForUser, unloadGroupAppExtender: unloadBackblazeB2ForUser, insertCompletedVideoExtender: uploadVideoToBackblazeB2, From 794fbba2a39107e8a822ec83b31987d08bcd8995 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 19 Sep 2023 20:38:08 -0700 Subject: [PATCH 008/112] Add more S3 Region locations based on #496 --- libs/uploaders/amazonS3.js | 190 +++++++++++++++++++++-------------- libs/uploaders/s3based.js | 196 ++++++++++++++++++++++--------------- 2 files changed, 235 insertions(+), 151 deletions(-) diff --git a/libs/uploaders/amazonS3.js b/libs/uploaders/amazonS3.js index c9f02149..95dd6497 100644 --- a/libs/uploaders/amazonS3.js +++ b/libs/uploaders/amazonS3.js @@ -315,79 +315,123 @@ module.exports = function(s,config,lang){ "default": "", "example": "", "possible": [ - { - "name": "US West (N. California)", - "value": "us-west-1" - }, - { - "name": "US West (Oregon)", - "value": "us-west-2" - }, - { - "name": "US East (Ohio)", - "value": "us-east-2" - }, - { - "name": "US East (N. Virginia)", - "value": "us-east-1" - }, - { - "name": "Asia Pacific (Mumbai)", - "value": "ap-south-1" - }, - { - "name": "Asia Pacific (Seoul)", - "value": "ap-northeast-2" - }, - { - "name": "Asia Pacific (Osaka-Local)**", - "value": "ap-northeast-3" - }, - { - "name": "Asia Pacific (Singapore)", - "value": "ap-southeast-1" - }, - { - "name": "Asia Pacific (Sydney)", - "value": "ap-southeast-2" - }, - { - "name": "Asia Pacific (Tokyo)", - "value": "ap-northeast-1" - }, - { - "name": "Canada (Central)", - "value": "ca-central-1" - }, - { - "name": "China (Beijing)", - "value": "cn-north-1" - }, - { - "name": "China (Ningxia)", - "value": "cn-northwest-1" - }, - { - "name": "EU (Frankfurt)", - "value": "eu-central-1" - }, - { - "name": "EU (Ireland)", - "value": "eu-west-1" - }, - { - "name": "EU (London)", - "value": "eu-west-2" - }, - { - "name": "EU (Paris)", - "value": "eu-west-3" - }, - { - "name": "South America (São Paulo)", - "value": "sa-east-1" - } - ] + { + "name": "US West (N. California)", + "value": "us-west-1" + }, + { + "name": "US West (Oregon)", + "value": "us-west-2" + }, + { + "name": "US East (Ohio)", + "value": "us-east-2" + }, + { + "name": "US East (N. Virginia)", + "value": "us-east-1" + }, + { + "name": "Canada (Central)", + "value": "ca-central-1" + }, + { + "name": "South America (São Paulo)", + "value": "sa-east-1" + }, + { + "name": "EU (Frankfurt)", + "value": "eu-central-1" + }, + { + "name": "EU (Ireland)", + "value": "eu-west-1" + }, + { + "name": "EU (London)", + "value": "eu-west-2" + }, + { + "name": "EU (Paris)", + "value": "eu-west-3" + }, + { + "name": "Europe (Milan)", + "value": "eu-south-1" + }, + { + "name": "Europe (Spain)", + "value": "eu-south-2" + }, + { + "name": "Europe (Zurich)", + "value": "eu-central-2" + }, + { + "name": "Asia Pacific (Mumbai)", + "value": "ap-south-1" + }, + { + "name": "Asia Pacific (Seoul)", + "value": "ap-northeast-2" + }, + { + "name": "Asia Pacific (Osaka-Local)**", + "value": "ap-northeast-3" + }, + { + "name": "Asia Pacific (Singapore)", + "value": "ap-southeast-1" + }, + { + "name": "Asia Pacific (Sydney)", + "value": "ap-southeast-2" + }, + { + "name": "Asia Pacific (Tokyo)", + "value": "ap-northeast-1" + }, + { + "name": "Asia Pacific (Hong Kong)", + "value": "ap-east-1" + }, + { + "name": "Asia Pacific (Hyderabad)", + "value": "ap-south-2" + }, + { + "name": "Asia Pacific (Jakarta)", + "value": "ap-southeast-3" + }, + { + "name": "Asia Pacific (Melbourne)", + "value": "ap-southeast-4" + }, + { + "name": "China (Beijing)", + "value": "cn-north-1" + }, + { + "name": "China (Ningxia)", + "value": "cn-northwest-1" + }, + { + "name": "Africa (Cape Town)", + "value": "af-south-1" + }, + { + "name": "Middle East (Bahrain)", + "value": "me-south-1" + }, + { + "name": "Middle East (UAE)", + "value": "me-central-1" + }, + { + "name": "il-central-1", + "value": "il-central-1" + } + ] }, { "hidden": true, diff --git a/libs/uploaders/s3based.js b/libs/uploaders/s3based.js index 34c053be..5ac2c52d 100644 --- a/libs/uploaders/s3based.js +++ b/libs/uploaders/s3based.js @@ -324,84 +324,124 @@ module.exports = function(s,config,lang){ "description": "", "default": "", "example": "", - "possible": [ - { - "name": lang['No Region'], - "value": "" - }, - { - "name": "US West 1", - "value": "us-west-1" - }, - { - "name": "US West 2", - "value": "us-west-2" - }, - { - "name": "US East 1", - "value": "us-east-1" - }, - { - "name": "US East 2", - "value": "us-east-2" - }, - { - "name": "Asia Pacific 1", - "value": "ap-south-1" - }, - { - "name": "Asia Pacific 2", - "value": "ap-northeast-2" - }, - { - "name": "Asia Pacific 3", - "value": "ap-northeast-3" - }, - { - "name": "Asia Pacific 4", - "value": "ap-southeast-1" - }, - { - "name": "Asia Pacific 5", - "value": "ap-southeast-2" - }, - { - "name": "Asia Pacific 6", - "value": "ap-northeast-1" - }, - { - "name": "Canada 1", - "value": "ca-central-1" - }, - { - "name": "China 1", - "value": "cn-north-1" - }, - { - "name": "China 1", - "value": "cn-northwest-1" - }, - { - "name": "EU 1", - "value": "eu-central-1" - }, - { - "name": "EU 2", - "value": "eu-west-1" - }, - { - "name": "EU 3", - "value": "eu-west-2" - }, - { - "name": "EU 4", - "value": "eu-west-3" - }, - { - "name": "South America 1", - "value": "sa-east-1" - } - ] + "possible": [ + { + "name": "US West (N. California)", + "value": "us-west-1" + }, + { + "name": "US West (Oregon)", + "value": "us-west-2" + }, + { + "name": "US East (Ohio)", + "value": "us-east-2" + }, + { + "name": "US East (N. Virginia)", + "value": "us-east-1" + }, + { + "name": "Canada (Central)", + "value": "ca-central-1" + }, + { + "name": "South America (São Paulo)", + "value": "sa-east-1" + }, + { + "name": "EU (Frankfurt)", + "value": "eu-central-1" + }, + { + "name": "EU (Ireland)", + "value": "eu-west-1" + }, + { + "name": "EU (London)", + "value": "eu-west-2" + }, + { + "name": "EU (Paris)", + "value": "eu-west-3" + }, + { + "name": "Europe (Milan)", + "value": "eu-south-1" + }, + { + "name": "Europe (Spain)", + "value": "eu-south-2" + }, + { + "name": "Europe (Zurich)", + "value": "eu-central-2" + }, + { + "name": "Asia Pacific (Mumbai)", + "value": "ap-south-1" + }, + { + "name": "Asia Pacific (Seoul)", + "value": "ap-northeast-2" + }, + { + "name": "Asia Pacific (Osaka-Local)**", + "value": "ap-northeast-3" + }, + { + "name": "Asia Pacific (Singapore)", + "value": "ap-southeast-1" + }, + { + "name": "Asia Pacific (Sydney)", + "value": "ap-southeast-2" + }, + { + "name": "Asia Pacific (Tokyo)", + "value": "ap-northeast-1" + }, + { + "name": "Asia Pacific (Hong Kong)", + "value": "ap-east-1" + }, + { + "name": "Asia Pacific (Hyderabad)", + "value": "ap-south-2" + }, + { + "name": "Asia Pacific (Jakarta)", + "value": "ap-southeast-3" + }, + { + "name": "Asia Pacific (Melbourne)", + "value": "ap-southeast-4" + }, + { + "name": "China (Beijing)", + "value": "cn-north-1" + }, + { + "name": "China (Ningxia)", + "value": "cn-northwest-1" + }, + { + "name": "Africa (Cape Town)", + "value": "af-south-1" + }, + { + "name": "Middle East (Bahrain)", + "value": "me-south-1" + }, + { + "name": "Middle East (UAE)", + "value": "me-central-1" + }, + { + "name": "il-central-1", + "value": "il-central-1" + } + ] }, { "hidden": true, From 5da361c6feaf668ec624876585669a812f0f2f6e Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 19 Sep 2023 20:39:10 -0700 Subject: [PATCH 009/112] re-add "No Region" to s3based --- libs/uploaders/s3based.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/uploaders/s3based.js b/libs/uploaders/s3based.js index 5ac2c52d..7894e08c 100644 --- a/libs/uploaders/s3based.js +++ b/libs/uploaders/s3based.js @@ -325,6 +325,10 @@ module.exports = function(s,config,lang){ "default": "", "example": "", "possible": [ + { + "name": lang['No Region'], + "value": "" + }, { "name": "US West (N. California)", "value": "us-west-1" From 38f5522627d9b1385b044828b0fd5e794b99d1a4 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 20 Sep 2023 11:31:50 -0700 Subject: [PATCH 010/112] "Video Access" and "User Logged Out" user logging --- languages/en_CA.json | 2 ++ libs/extenders.js | 2 ++ libs/user.js | 1 + libs/user/logger.js | 31 +++++++++++++++++++++++++++++++ libs/webServerPaths.js | 25 +++++++++++++++++++------ 5 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 libs/user/logger.js diff --git a/languages/en_CA.json b/languages/en_CA.json index f4b4e075..e1923e31 100644 --- a/languages/en_CA.json +++ b/languages/en_CA.json @@ -145,6 +145,7 @@ "Input Settings": "Input Settings", "Connection": "Connection", "Video Set": "Video Set", + "Video Accessed": "Video Accessed", "notEnoughFramesText1": "Not Enough Frames for compilation.", "Allow API Trigger": "Allow API Trigger", "When Detector is Off": "When Detector is Off", @@ -480,6 +481,7 @@ "Database": "Database", "Database Not Found": "Database Not Found", "User Not Found": "User Not Found", + "User Logged Out": "User Logged Out", "Not Found": "Not Found", "Save Links to Database": "Save Links to Database", "Uploaders": "Uploaders", diff --git a/libs/extenders.js b/libs/extenders.js index 51535c93..7cf3595d 100644 --- a/libs/extenders.js +++ b/libs/extenders.js @@ -25,6 +25,8 @@ module.exports = function(s,config){ createExtension(`beforeAccountSave`) createExtension(`onTwoFactorAuthCodeNotification`) createExtension(`onStalePurgeLock`) + createExtension(`onVideoAccess`) + createExtension(`onLogout`) ////// EVENTS ////// createExtension(`onEventTrigger`) createExtension(`onEventTriggerBeforeFilter`) diff --git a/libs/user.js b/libs/user.js index ea580253..3187c250 100644 --- a/libs/user.js +++ b/libs/user.js @@ -18,6 +18,7 @@ module.exports = function(s,config,lang){ deleteCloudTimelapseFrames, resetAllStorageCounters, } = require("./user/utils.js")(s,config,lang); + require("./user/logger.js")(s,config,lang) let purgeDiskGroup = () => {} const runQuery = async.queue(function(groupKey, callback) { purgeDiskGroup(groupKey,callback) diff --git a/libs/user/logger.js b/libs/user/logger.js new file mode 100644 index 00000000..f26c25a6 --- /dev/null +++ b/libs/user/logger.js @@ -0,0 +1,31 @@ +module.exports = function(s,config,lang){ + s.onVideoAccess((videoRow,user,groupKey,monitorId,ip) => { + s.userLog({ + ke: groupKey, + mid: '$USER', + },{ + type: lang['Video Accessed'], + msg: { + user: { + mail: user.mail, + uid: user.uid, + ip, + }, + video: videoRow + } + }) + }) + s.onLogout((user,groupKey,userId,ip) => { + s.userLog({ + ke: groupKey, + mid: '$USER', + },{ + type: lang['User Logged Out'], + msg: { + mail: user.mail, + uid: user.uid, + ip, + } + }) + }) +} diff --git a/libs/webServerPaths.js b/libs/webServerPaths.js index 104fa3b3..25db79b2 100644 --- a/libs/webServerPaths.js +++ b/libs/webServerPaths.js @@ -107,9 +107,18 @@ module.exports = function(s,config,lang,app,io){ * API : Logout */ app.get(config.webPaths.apiPrefix+':auth/logout/:ke/:id', function (req,res){ - if(s.group[req.params.ke] && s.group[req.params.ke].users[req.params.auth] && s.group[req.params.ke].users[req.params.auth].details){ - delete(s.api[req.params.auth]); - delete(s.group[req.params.ke].users[req.params.auth]); + const groupKey = req.params.ke + const userId = req.params.id + const authToken = req.params.auth + const user = s.group[groupKey] && s.group[groupKey].users[authToken] && s.group[groupKey].users[authToken].details ? s.group[groupKey].users[authToken] : null; + if(user){ + const clientIp = s.getClientIp(req) + const user = s.group[groupKey].users[authToken] + s.onLogoutExtensions.forEach((extender) => { + extender(user,groupKey,userId,clientIp) + }); + delete(s.api[authToken]); + delete(s.group[groupKey].users[authToken]); s.knexQuery({ action: "update", table: "Users", @@ -117,9 +126,9 @@ module.exports = function(s,config,lang,app,io){ auth: '', }, where: [ - ['auth','=',req.params.auth], - ['ke','=',req.params.ke], - ['uid','=',req.params.id], + ['auth','=',authToken], + ['ke','=',groupKey], + ['uid','=',userId], ] }) res.end(s.prettyPrint({ok:true,msg:'You have been logged out, session key is now inactive.'})) @@ -1454,6 +1463,10 @@ module.exports = function(s,config,lang,app,io){ }else{ s.streamMp4FileOverHttp(filePath,req,res,!!req.query.pureStream) } + const clientIp = s.getClientIp(req) + s.onVideoAccessExtensions.forEach((extender) => { + extender(videoRow,user,groupKey,monitorId,clientIp) + }) }else{ s.closeJsonResponse(res,{ ok: false, From a70a4b00bf8279520758709404e2d59a972305ac Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 20 Sep 2023 11:32:54 -0700 Subject: [PATCH 011/112] better tree draw Log items in log page --- web/assets/css/dashboard.css | 5 +++++ web/assets/js/bs5.dashboard-base.js | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/web/assets/css/dashboard.css b/web/assets/css/dashboard.css index e007b649..7a61b7fd 100644 --- a/web/assets/css/dashboard.css +++ b/web/assets/css/dashboard.css @@ -431,3 +431,8 @@ ul.squeeze { flex-direction: column; height: 100vh; } + +.log-item .msg-tree > ul { + list-style: none; + padding-left: 0; +} diff --git a/web/assets/js/bs5.dashboard-base.js b/web/assets/js/bs5.dashboard-base.js index 0edb4b92..a8d7cc1c 100644 --- a/web/assets/js/bs5.dashboard-base.js +++ b/web/assets/js/bs5.dashboard-base.js @@ -327,11 +327,13 @@ function compileConnectUrl(options){ function jsonToHtmlBlock(target){ var html = '' if(target instanceof Object){ + var html = '
    ' $.each(target,function(key,value){ - html += `
    - ${key} : ${jsonToHtmlBlock(value)} -
    ` + html += ` +
  • ${key} : ${jsonToHtmlBlock(value)}
    • + ` }) + html += '
' }else{ html += `${target}` } @@ -836,7 +838,7 @@ function buildLogRow(v){ ${humanMonitorName}${v.info && v.info.type ? v.info.type : v.mid}
-
${jsonToHtmlBlock(v.info.msg)}
+
${jsonToHtmlBlock(v.info.msg)}
${formattedTime(v.time)} From 01f498c89d6969cbbdac260e2e8b29e29ad75205 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 20 Sep 2023 12:15:13 -0700 Subject: [PATCH 012/112] add event markers to timeline strip --- web/assets/js/bs5.timeline.js | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/web/assets/js/bs5.timeline.js b/web/assets/js/bs5.timeline.js index d4b2cf5f..6b342ba4 100644 --- a/web/assets/js/bs5.timeline.js +++ b/web/assets/js/bs5.timeline.js @@ -189,17 +189,28 @@ $(document).ready(function(){ } } function formatVideosForTimeline(videos){ - var formattedVideos = (videos || []).map((item) => { - var blockColor = timeStripItemColors[item.mid]; + var formattedVideos = []; + (videos || []).forEach((video) => { + var blockColor = timeStripItemColors[video.mid]; ++timeStripItemIncrement; - return { + formattedVideos.push({ id: timeStripItemIncrement, content: ``, style: `background-color: ${blockColor};border-color: ${blockColor}`, - start: item.time, - end: item.end, + start: video.time, + end: video.end, group: 1 - } + }) + video.events.forEach((theEvent) => { + ++timeStripItemIncrement; + formattedVideos.push({ + id: timeStripItemIncrement, + content: ``, + style: `background-color: yellow;border-color: yellow`, + start: theEvent.time, + group: 1 + }) + }) }); return formattedVideos } From 3a3e75c9b8ee0b93c7f501abad00b0efe614f5c4 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 20 Sep 2023 12:23:42 -0700 Subject: [PATCH 013/112] Allow more rows per page on Videos table --- web/assets/js/bs5.videosTable.js | 1 + 1 file changed, 1 insertion(+) diff --git a/web/assets/js/bs5.videosTable.js b/web/assets/js/bs5.videosTable.js index d674743a..c803b936 100644 --- a/web/assets/js/bs5.videosTable.js +++ b/web/assets/js/bs5.videosTable.js @@ -94,6 +94,7 @@ $(document).ready(function(e){ }, pagination: true, search: true, + pageList: [10, 25, 50, 100, 1000, 2000], columns: [ { field: 'mid', From 2fe9cd1831ab9bf3b44b3abdf4aa5adcfe0e42f4 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Sun, 24 Sep 2023 10:13:19 +0300 Subject: [PATCH 014/112] docker images upgrade --- .gitlab-ci.yml | 87 ++++++++++++++++++++++++++++ Docker/install_dependencies.sh | 85 +++++++++++++++++++++++++++ Dockerfile | 91 +++-------------------------- Dockerfile.arm32v7 | 91 +++-------------------------- Dockerfile.nodb | 97 ------------------------------- Dockerfile.nvidia | 101 ++++----------------------------- 6 files changed, 200 insertions(+), 352 deletions(-) create mode 100644 Docker/install_dependencies.sh delete mode 100644 Dockerfile.nodb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79d095a1..ff37bb9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,6 +39,35 @@ docker-latest-build: exists: - Dockerfile +docker-latest-no-db-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag="latest-no-db" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest-no-db'" + else + tag="latest-no-db:$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + docker-arm32v7-build: # Use the official docker image. image: docker:latest @@ -68,6 +97,35 @@ docker-arm32v7-build: exists: - Dockerfile.arm32v7 +docker-arm32v7-no-db-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag=":arm32v7-no-db" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm32v7-no-db'" + else + tag=":arm32v7-no-db-$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm32v7 + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile.arm32v7 + docker-nvidia-build: # Use the official docker image. @@ -97,3 +155,32 @@ docker-nvidia-build: - if: $CI_COMMIT_BRANCH exists: - Dockerfile.nvidia + +docker-nvidia-no-db-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag=":nvidia-no-db" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'nvidia-no-db'" + else + tag=":nvidia-no-db-$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile.nvidia diff --git a/Docker/install_dependencies.sh b/Docker/install_dependencies.sh new file mode 100644 index 00000000..de213126 --- /dev/null +++ b/Docker/install_dependencies.sh @@ -0,0 +1,85 @@ +#!/bin/sh +mkdir -p /etc/apt/keyrings + +apt update -y --fix-missing +apt upgrade -y +apt update -y --fix-missing + +apt install -y \ + wget \ + curl \ + net-tools \ + software-properties-common \ + libfreetype6-dev \ + libgnutls28-dev \ + libmp3lame-dev \ + libass-dev \ + libogg-dev \ + libtheora-dev \ + libvorbis-dev \ + libvpx-dev \ + libwebp-dev \ + libssh2-1-dev \ + libopus-dev \ + librtmp-dev \ + libx264-dev \ + libx265-dev \ + yasm \ + build-essential \ + bzip2 \ + coreutils \ + procps \ + gnutls-bin \ + nasm \ + tar \ + x264 \ + ffmpeg \ + git \ + make \ + g++ \ + gcc \ + pkg-config \ + python3 \ + tar \ + sudo \ + xz-utils \ + ca-certificates \ + gnupg + +if [ "$INSTALL_NODE" = "true" ] ; then set -ex; \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg ; \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list ; \ + apt update -y --fix-missing ; \ + apt upgrade -y ; \ + apt install nodejs -y ; fi + +node -v +npm -v + +npm i npm@latest pm2 pg -g --save +npm install --unsafe-perm + +chmod 777 /home/Shinobi +chmod -R 777 /home/Shinobi/plugins +chmod -f +x /home/Shinobi/Docker/init.sh + +sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh + +ffmpeg -version + +# Install MariaDB server... the debian way +if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ + { \ + echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ + echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ + } | debconf-set-selections; \ + mkdir -p /var/lib/mysql; \ + apt-get update; \ + apt-get install -y \ + "mariadb-server" \ + socat \ + ; \ + find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; \ + sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi diff --git a/Dockerfile b/Dockerfile index a94abea0..4414a790 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -FROM node:16-buster-slim +FROM node:18-bullseye-slim + +ARG DEBIAN_FRONTEND=noninteractive \ + VAR_EXCLUDE_DB=false ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -15,97 +18,19 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - + DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN chmod 777 /home/Shinobi -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ +COPY . ./ -# Copy default configuration files -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi +RUN sh /home/Shinobi/Docker/install_dependencies.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] EXPOSE 8080 443 21 25 -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] +ENTRYPOINT ["sh", "/home/Shinobi/Docker/init.sh"] CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 index 2e724f77..9957f392 100644 --- a/Dockerfile.arm32v7 +++ b/Dockerfile.arm32v7 @@ -1,4 +1,7 @@ -FROM arm32v7/node:16-buster-slim +FROM arm32v7/node:18-bullseye-slim + +ARG DEBIAN_FRONTEND=noninteractive \ + VAR_EXCLUDE_DB=false ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -15,95 +18,19 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm && \ - apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y zip - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - + DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install pm2 -g && \ - npm install --unsafe-perm -COPY ./Docker/pm2.yml ./ +COPY . ./ -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh +RUN sh /home/Shinobi/Docker/install_dependencies.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] -EXPOSE 8080 +EXPOSE 8080 443 21 25 ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] -CMD [ "pm2-docker", "pm2.yml" ] +CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.nodb b/Dockerfile.nodb deleted file mode 100644 index 8648484f..00000000 --- a/Dockerfile.nodb +++ /dev/null @@ -1,97 +0,0 @@ -FROM node:16-buster-slim - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=true -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y zip - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ - -RUN npm i pg --save - -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia index d6221971..886b92b9 100644 --- a/Dockerfile.nvidia +++ b/Dockerfile.nvidia @@ -1,4 +1,8 @@ -FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu20.04 +FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04 + +ARG DEBIAN_FRONTEND=noninteractive \ + VAR_EXCLUDE_DB=false \ + VAR_INSTALL_NODE=true ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -15,104 +19,21 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Additional packages to support NodeJS as the base build is NVidia CUDA -RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - -RUN apt install nodejs -y - -RUN node -v -RUN npm -v - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - + DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB \ + NODE_MAJOR=18 \ + INSTALL_NODE=$VAR_INSTALL_NODE WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ +COPY . ./ -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi +RUN sh /home/Shinobi/Docker/install_dependencies.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] EXPOSE 8080 443 21 25 -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] +ENTRYPOINT ["sh", "/home/Shinobi/Docker/init.sh"] CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] From 6701b47eeb059a6b560b25018ac8591426a4671e Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Mon, 25 Sep 2023 13:28:48 +0300 Subject: [PATCH 015/112] add support for arm64v8 --- .gitlab-ci.yml | 58 ++++++++++++++++++++++++++++++++++++++++++++++ Dockerfile.arm64v8 | 36 ++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 Dockerfile.arm64v8 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ff37bb9b..b82dbe4a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -126,6 +126,64 @@ docker-arm32v7-no-db-build: exists: - Dockerfile.arm32v7 +docker-arm64v8-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag=":arm64v8" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm64v8'" + else + tag=":arm64v8-$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm64v8 + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile.arm64v8 + +docker-arm64v8-no-db-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag=":arm64v8-no-db" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm64v8-no-db'" + else + tag=":arm64v8-no-db-$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm64v8 + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile.arm64v8 + docker-nvidia-build: # Use the official docker image. diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 new file mode 100644 index 00000000..66b51fa2 --- /dev/null +++ b/Dockerfile.arm64v8 @@ -0,0 +1,36 @@ +FROM arm64v8/node:18-bullseye-slim + +ARG DEBIAN_FRONTEND=noninteractive \ + VAR_EXCLUDE_DB=false + +ENV DB_USER=majesticflame \ + DB_PASSWORD='' \ + DB_HOST='localhost' \ + DB_DATABASE=ccio \ + DB_PORT=3306 \ + DB_TYPE='mysql' \ + SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ + PLUGIN_KEYS='{}' \ + SSL_ENABLED='false' \ + SSL_COUNTRY='CA' \ + SSL_STATE='BC' \ + SSL_LOCATION='Vancouver' \ + SSL_ORGANIZATION='Shinobi Systems' \ + SSL_ORGANIZATION_UNIT='IT Department' \ + SSL_COMMON_NAME='nvr.ninja' \ + DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB + +WORKDIR /home/Shinobi +COPY . ./ + +RUN sh /home/Shinobi/Docker/install_dependencies.sh + +VOLUME ["/home/Shinobi/videos"] +VOLUME ["/home/Shinobi/libs/customAutoLoad"] +VOLUME ["/config"] + +EXPOSE 8080 443 21 25 + +ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] + +CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] From 8228487649b627b8f22f0edfea311edc185a8ccb Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Tue, 26 Sep 2023 09:27:21 +0300 Subject: [PATCH 016/112] fix default distribution no-db build tag name --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b82dbe4a..8e0a3cce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,10 +52,10 @@ docker-latest-no-db-build: script: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag="latest-no-db" + tag=":no-db" echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest-no-db'" else - tag="latest-no-db:$CI_COMMIT_REF_SLUG" + tag=":no-db-$CI_COMMIT_REF_SLUG" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . From db94ab00fcf124536507e48f0d5fb9d6151e199e Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 08:57:30 +0300 Subject: [PATCH 017/112] improve CI script --- .gitlab-ci.yml | 62 +++++++++++++++++++++------------- Docker/install_dependencies.sh | 3 +- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e0a3cce..53efe22f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,10 +52,10 @@ docker-latest-no-db-build: script: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":no-db" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest-no-db'" + tag=":latest-no-db" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" else - tag=":no-db-$CI_COMMIT_REF_SLUG" + tag=":${CI_COMMIT_REF_SLUG}-no-db" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . @@ -82,9 +82,9 @@ docker-arm32v7-build: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then tag=":arm32v7" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm32v7'" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" else - tag=":arm32v7-$CI_COMMIT_REF_SLUG" + tag=":${CI_COMMIT_REF_SLUG}-arm32v7" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm32v7 @@ -111,9 +111,9 @@ docker-arm32v7-no-db-build: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then tag=":arm32v7-no-db" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm32v7-no-db'" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" else - tag=":arm32v7-no-db-$CI_COMMIT_REF_SLUG" + tag=":${CI_COMMIT_REF_SLUG}-arm32v7-no-db" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm32v7 @@ -140,9 +140,9 @@ docker-arm64v8-build: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then tag=":arm64v8" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm64v8'" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" else - tag=":arm64v8-$CI_COMMIT_REF_SLUG" + tag=":${CI_COMMIT_REF_SLUG}-arm64v8" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm64v8 @@ -169,9 +169,9 @@ docker-arm64v8-no-db-build: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then tag=":arm64v8-no-db" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm64v8-no-db'" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" else - tag=":arm64v8-no-db-$CI_COMMIT_REF_SLUG" + tag=":${CI_COMMIT_REF_SLUG}-arm64v8-no-db" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm64v8 @@ -199,9 +199,9 @@ docker-nvidia-build: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then tag=":nvidia" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'nvidia'" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" else - tag=":nvidia-$CI_COMMIT_REF_SLUG" + tag=":${CI_COMMIT_REF_SLUG}-nvidia" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia @@ -226,19 +226,35 @@ docker-nvidia-no-db-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":nvidia-no-db" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'nvidia-no-db'" - else - tag=":nvidia-no-db-$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + arch="nvidia" + branch="" + no_db=true + no_db_tag = "" + docker_file_ext="" + + if [[ "${arch}" != "" ]]; then + docker_file_ext=".${arch}" fi - - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia - - docker push "$CI_REGISTRY_IMAGE${tag}" + + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then + branch="${CI_COMMIT_REF_SLUG}-" + fi + + if [[ "${no_db}" == "true" ]]; then + no_db_tag="-no-db" + fi + + tag=":${branch}${arch}${no_db_tag}" + + image_tag="$CI_REGISTRY_IMAGE${tag}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Tag: ${tag}, Image: ${image_tag}" + + - docker build --pull --build-arg VAR_EXCLUDE_DB="${no_db}" -t "${image_tag}" . -f "Dockerfile${docker_file_ext}" + - docker push "${image_tag}" # Run this job in a branch where a Dockerfile exists rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile.nvidia diff --git a/Docker/install_dependencies.sh b/Docker/install_dependencies.sh index de213126..b1622cc5 100644 --- a/Docker/install_dependencies.sh +++ b/Docker/install_dependencies.sh @@ -44,7 +44,8 @@ apt install -y \ sudo \ xz-utils \ ca-certificates \ - gnupg + gnupg \ + apt-utils if [ "$INSTALL_NODE" = "true" ] ; then set -ex; \ curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg ; \ From a2af613191b3927b0e85c9a21a0b259c5bc6ff0a Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 09:00:11 +0300 Subject: [PATCH 018/112] fix script #1 --- .gitlab-ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53efe22f..b7cbbcb5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -226,28 +226,28 @@ docker-nvidia-no-db-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - arch="nvidia" - branch="" - no_db=true - no_db_tag = "" - docker_file_ext="" + arch="nvidia" ; + branch="" ; + no_db=true ; + no_db_tag = "" ; + docker_file_ext="" ; if [[ "${arch}" != "" ]]; then docker_file_ext=".${arch}" - fi + fi if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then branch="${CI_COMMIT_REF_SLUG}-" - fi + fi if [[ "${no_db}" == "true" ]]; then no_db_tag="-no-db" fi - tag=":${branch}${arch}${no_db_tag}" + tag=":${branch}${arch}${no_db_tag}" ; - image_tag="$CI_REGISTRY_IMAGE${tag}" - echo "Running on branch '${CI_COMMIT_BRANCH}', Tag: ${tag}, Image: ${image_tag}" + image_tag="$CI_REGISTRY_IMAGE${tag}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Tag: ${tag}, Image: ${image_tag}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${no_db}" -t "${image_tag}" . -f "Dockerfile${docker_file_ext}" - docker push "${image_tag}" From e585fe9d81503e4e76c25dd1eb7db710ef727c5f Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 09:13:19 +0300 Subject: [PATCH 019/112] fix script #2 --- .gitlab-ci.yml | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7cbbcb5..79dee843 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,6 +218,10 @@ docker-nvidia-no-db-build: # Use the official docker image. image: docker:latest stage: build + variables: + ARCH_TYPE: "nvidia" + INCLUDE_DB: false + DOCKER_FILE: "Dockerfile.nvidia" services: - docker:dind before_script: @@ -226,31 +230,22 @@ docker-nvidia-no-db-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - arch="nvidia" ; - branch="" ; - no_db=true ; - no_db_tag = "" ; - docker_file_ext="" ; - - if [[ "${arch}" != "" ]]; then - docker_file_ext=".${arch}" - fi - + export BRANCH_PREFIX="" ; + export NO_DB_SUFIX="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - branch="${CI_COMMIT_REF_SLUG}-" + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; fi - if [[ "${no_db}" == "true" ]]; then - no_db_tag="-no-db" + if [[ "${INCLUDE_DB}" == "true" ]]; then + export NO_DB_SUFIX="-no-db" ; fi - tag=":${branch}${arch}${no_db_tag}" ; - - image_tag="$CI_REGISTRY_IMAGE${tag}" ; - echo "Running on branch '${CI_COMMIT_BRANCH}', Tag: ${tag}, Image: ${image_tag}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${no_db}" -t "${image_tag}" . -f "Dockerfile${docker_file_ext}" - - docker push "${image_tag}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker push "${IMAGE_NAME}" # Run this job in a branch where a Dockerfile exists rules: # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" From cf205f3e9ad2a072dcbb6d0085395948aa7c439f Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 09:14:28 +0300 Subject: [PATCH 020/112] fix script #3 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79dee843..f9f27f88 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -220,7 +220,7 @@ docker-nvidia-no-db-build: stage: build variables: ARCH_TYPE: "nvidia" - INCLUDE_DB: false + INCLUDE_DB: "false" DOCKER_FILE: "Dockerfile.nvidia" services: - docker:dind From 9b27034acb30c8cb07a837bffeedc24d44428b3a Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 09:21:01 +0300 Subject: [PATCH 021/112] test all --- .gitlab-ci.yml | 235 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 156 insertions(+), 79 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9f27f88..c167d780 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,10 @@ docker-latest-build: # Use the official docker image. image: docker:latest stage: build + variables: + ARCH_TYPE: "" + INCLUDE_DB: "true" + DOCKER_FILE: "Dockerfile" services: - docker:dind before_script: @@ -22,27 +26,38 @@ docker-latest-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag="" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" - else - tag=":$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + export BRANCH_PREFIX="" ; + export NO_DB_SUFIX="" ; + + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + fi + + if [[ "${INCLUDE_DB}" == "false" ]]; then + export NO_DB_SUFIX="-no-db" ; fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . - - docker push "$CI_REGISTRY_IMAGE${tag}" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker push "${IMAGE_NAME}" # Run this job in a branch where a Dockerfile exists rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile + - $DOCKER_FILE docker-latest-no-db-build: # Use the official docker image. image: docker:latest stage: build + variables: + ARCH_TYPE: "" + INCLUDE_DB: "false" + DOCKER_FILE: "Dockerfile" services: - docker:dind before_script: @@ -51,27 +66,38 @@ docker-latest-no-db-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":latest-no-db" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" - else - tag=":${CI_COMMIT_REF_SLUG}-no-db" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + export BRANCH_PREFIX="" ; + export NO_DB_SUFIX="" ; + + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + fi + + if [[ "${INCLUDE_DB}" == "false" ]]; then + export NO_DB_SUFIX="-no-db" ; fi - - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . - - docker push "$CI_REGISTRY_IMAGE${tag}" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker push "${IMAGE_NAME}" # Run this job in a branch where a Dockerfile exists rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile + - $DOCKER_FILE docker-arm32v7-build: # Use the official docker image. image: docker:latest stage: build + variables: + ARCH_TYPE: "arm32v7" + INCLUDE_DB: "true" + DOCKER_FILE: "Dockerfile.arm32v7" services: - docker:dind before_script: @@ -80,27 +106,38 @@ docker-arm32v7-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":arm32v7" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" - else - tag=":${CI_COMMIT_REF_SLUG}-arm32v7" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + export BRANCH_PREFIX="" ; + export NO_DB_SUFIX="" ; + + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + fi + + if [[ "${INCLUDE_DB}" == "false" ]]; then + export NO_DB_SUFIX="-no-db" ; fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm32v7 - - docker push "$CI_REGISTRY_IMAGE${tag}" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker push "${IMAGE_NAME}" # Run this job in a branch where a Dockerfile exists rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.arm32v7 + - $DOCKER_FILE docker-arm32v7-no-db-build: # Use the official docker image. image: docker:latest stage: build + variables: + ARCH_TYPE: "arm32v7" + INCLUDE_DB: "false" + DOCKER_FILE: "Dockerfile.arm32v7" services: - docker:dind before_script: @@ -109,27 +146,38 @@ docker-arm32v7-no-db-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":arm32v7-no-db" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" - else - tag=":${CI_COMMIT_REF_SLUG}-arm32v7-no-db" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + export BRANCH_PREFIX="" ; + export NO_DB_SUFIX="" ; + + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + fi + + if [[ "${INCLUDE_DB}" == "false" ]]; then + export NO_DB_SUFIX="-no-db" ; fi - - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm32v7 - - docker push "$CI_REGISTRY_IMAGE${tag}" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker push "${IMAGE_NAME}" # Run this job in a branch where a Dockerfile exists rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.arm32v7 + - $DOCKER_FILE docker-arm64v8-build: # Use the official docker image. image: docker:latest stage: build + variables: + ARCH_TYPE: "arm64v8" + INCLUDE_DB: "true" + DOCKER_FILE: "Dockerfile.arm64v8" services: - docker:dind before_script: @@ -138,27 +186,38 @@ docker-arm64v8-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":arm64v8" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" - else - tag=":${CI_COMMIT_REF_SLUG}-arm64v8" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + export BRANCH_PREFIX="" ; + export NO_DB_SUFIX="" ; + + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + fi + + if [[ "${INCLUDE_DB}" == "false" ]]; then + export NO_DB_SUFIX="-no-db" ; fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm64v8 - - docker push "$CI_REGISTRY_IMAGE${tag}" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker push "${IMAGE_NAME}" # Run this job in a branch where a Dockerfile exists rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.arm64v8 + - $DOCKER_FILE docker-arm64v8-no-db-build: # Use the official docker image. image: docker:latest stage: build + variables: + ARCH_TYPE: "arm64v8" + INCLUDE_DB: "false" + DOCKER_FILE: "Dockerfile.arm64v8" services: - docker:dind before_script: @@ -167,28 +226,39 @@ docker-arm64v8-no-db-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":arm64v8-no-db" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" - else - tag=":${CI_COMMIT_REF_SLUG}-arm64v8-no-db" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + export BRANCH_PREFIX="" ; + export NO_DB_SUFIX="" ; + + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + fi + + if [[ "${INCLUDE_DB}" == "false" ]]; then + export NO_DB_SUFIX="-no-db" ; fi - - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm64v8 - - docker push "$CI_REGISTRY_IMAGE${tag}" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker push "${IMAGE_NAME}" # Run this job in a branch where a Dockerfile exists rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.arm64v8 + - $DOCKER_FILE docker-nvidia-build: # Use the official docker image. image: docker:latest stage: build + variables: + ARCH_TYPE: "nvidia" + INCLUDE_DB: "true" + DOCKER_FILE: "Dockerfile.nvidia" services: - docker:dind before_script: @@ -197,22 +267,29 @@ docker-nvidia-build: # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":nvidia" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = '${tag}'" - else - tag=":${CI_COMMIT_REF_SLUG}-nvidia" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + export BRANCH_PREFIX="" ; + export NO_DB_SUFIX="" ; + + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + fi + + if [[ "${INCLUDE_DB}" == "false" ]]; then + export NO_DB_SUFIX="-no-db" ; fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia - - docker push "$CI_REGISTRY_IMAGE${tag}" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker push "${IMAGE_NAME}" # Run this job in a branch where a Dockerfile exists rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.nvidia + - $DOCKER_FILE docker-nvidia-no-db-build: # Use the official docker image. @@ -237,7 +314,7 @@ docker-nvidia-no-db-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; fi - if [[ "${INCLUDE_DB}" == "true" ]]; then + if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -252,4 +329,4 @@ docker-nvidia-no-db-build: # when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.nvidia + - $DOCKER_FILE From 23d1bc07ed416e899cf4c2fc2be2150d8a192bde Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 09:29:53 +0300 Subject: [PATCH 022/112] fix branch separator --- .gitlab-ci.yml | 88 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c167d780..a356989a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,16 +28,21 @@ docker-latest-build: - | export BRANCH_PREFIX="" ; export NO_DB_SUFIX="" ; - + export BRANCH_SEP="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + if [[ "${ARCH_TYPE}" != "" ]]; then + export BRANCH_SEP="-" ; + fi + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" @@ -68,16 +73,21 @@ docker-latest-no-db-build: - | export BRANCH_PREFIX="" ; export NO_DB_SUFIX="" ; - + export BRANCH_SEP="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + if [[ "${ARCH_TYPE}" != "" ]]; then + export BRANCH_SEP="-" ; + fi + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" @@ -108,16 +118,21 @@ docker-arm32v7-build: - | export BRANCH_PREFIX="" ; export NO_DB_SUFIX="" ; - + export BRANCH_SEP="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + if [[ "${ARCH_TYPE}" != "" ]]; then + export BRANCH_SEP="-" ; + fi + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" @@ -148,16 +163,21 @@ docker-arm32v7-no-db-build: - | export BRANCH_PREFIX="" ; export NO_DB_SUFIX="" ; - + export BRANCH_SEP="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + if [[ "${ARCH_TYPE}" != "" ]]; then + export BRANCH_SEP="-" ; + fi + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" @@ -188,16 +208,21 @@ docker-arm64v8-build: - | export BRANCH_PREFIX="" ; export NO_DB_SUFIX="" ; - + export BRANCH_SEP="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + if [[ "${ARCH_TYPE}" != "" ]]; then + export BRANCH_SEP="-" ; + fi + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" @@ -228,16 +253,21 @@ docker-arm64v8-no-db-build: - | export BRANCH_PREFIX="" ; export NO_DB_SUFIX="" ; - + export BRANCH_SEP="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + if [[ "${ARCH_TYPE}" != "" ]]; then + export BRANCH_SEP="-" ; + fi + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" @@ -269,16 +299,21 @@ docker-nvidia-build: - | export BRANCH_PREFIX="" ; export NO_DB_SUFIX="" ; - + export BRANCH_SEP="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + if [[ "${ARCH_TYPE}" != "" ]]; then + export BRANCH_SEP="-" ; + fi + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" @@ -309,16 +344,21 @@ docker-nvidia-no-db-build: - | export BRANCH_PREFIX="" ; export NO_DB_SUFIX="" ; - + export BRANCH_SEP="" ; + if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}-" ; + export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi if [[ "${INCLUDE_DB}" == "false" ]]; then export NO_DB_SUFIX="-no-db" ; fi - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" ; + if [[ "${ARCH_TYPE}" != "" ]]; then + export BRANCH_SEP="-" ; + fi + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" From a0944e8a99b3b330ca4f8736ef3b154fcde72e7a Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 09:39:28 +0300 Subject: [PATCH 023/112] clean comments, fix exclude DB --- .gitlab-ci.yml | 141 +++++++++++++++++-------------------------------- 1 file changed, 48 insertions(+), 93 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a356989a..05b8f29e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,32 +1,17 @@ -# This file is a template, and might need editing before it works on your project. -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml - -# Build a Docker image with CI/CD and push to the GitLab registry. -# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html -# -# This template uses one generic job with conditional builds -# for the default branch and all other (MR) branches. - docker-latest-build: - # Use the official docker image. image: docker:latest stage: build variables: ARCH_TYPE: "" - INCLUDE_DB: "true" + EXCLUDE_DB: "false" DOCKER_FILE: "Dockerfile" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - export BRANCH_PREFIX="" ; + export BRANCH_PREFIX="latest" ; export NO_DB_SUFIX="" ; export BRANCH_SEP="" ; @@ -34,7 +19,7 @@ docker-latest-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi - if [[ "${INCLUDE_DB}" == "false" ]]; then + if [[ "${EXCLUDE_DB}" == "true" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -45,33 +30,29 @@ docker-latest-build: export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" - # Run this job in a branch where a Dockerfile exists rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE docker-latest-no-db-build: - # Use the official docker image. image: docker:latest stage: build variables: ARCH_TYPE: "" - INCLUDE_DB: "false" + EXCLUDE_DB: "true" DOCKER_FILE: "Dockerfile" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - export BRANCH_PREFIX="" ; + export BRANCH_PREFIX="latest" ; export NO_DB_SUFIX="" ; export BRANCH_SEP="" ; @@ -79,7 +60,7 @@ docker-latest-no-db-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi - if [[ "${INCLUDE_DB}" == "false" ]]; then + if [[ "${EXCLUDE_DB}" == "true" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -90,33 +71,29 @@ docker-latest-no-db-build: export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" - # Run this job in a branch where a Dockerfile exists rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE docker-arm32v7-build: - # Use the official docker image. image: docker:latest stage: build variables: ARCH_TYPE: "arm32v7" - INCLUDE_DB: "true" + EXCLUDE_DB: "false" DOCKER_FILE: "Dockerfile.arm32v7" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - export BRANCH_PREFIX="" ; + export BRANCH_PREFIX="latest" ; export NO_DB_SUFIX="" ; export BRANCH_SEP="" ; @@ -124,7 +101,7 @@ docker-arm32v7-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi - if [[ "${INCLUDE_DB}" == "false" ]]; then + if [[ "${EXCLUDE_DB}" == "true" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -135,33 +112,29 @@ docker-arm32v7-build: export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" - # Run this job in a branch where a Dockerfile exists rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE docker-arm32v7-no-db-build: - # Use the official docker image. image: docker:latest stage: build variables: ARCH_TYPE: "arm32v7" - INCLUDE_DB: "false" + EXCLUDE_DB: "true" DOCKER_FILE: "Dockerfile.arm32v7" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - export BRANCH_PREFIX="" ; + export BRANCH_PREFIX="latest" ; export NO_DB_SUFIX="" ; export BRANCH_SEP="" ; @@ -169,7 +142,7 @@ docker-arm32v7-no-db-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi - if [[ "${INCLUDE_DB}" == "false" ]]; then + if [[ "${EXCLUDE_DB}" == "true" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -180,33 +153,29 @@ docker-arm32v7-no-db-build: export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" - # Run this job in a branch where a Dockerfile exists rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE docker-arm64v8-build: - # Use the official docker image. image: docker:latest stage: build variables: ARCH_TYPE: "arm64v8" - INCLUDE_DB: "true" + EXCLUDE_DB: "false" DOCKER_FILE: "Dockerfile.arm64v8" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - export BRANCH_PREFIX="" ; + export BRANCH_PREFIX="latest" ; export NO_DB_SUFIX="" ; export BRANCH_SEP="" ; @@ -214,7 +183,7 @@ docker-arm64v8-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi - if [[ "${INCLUDE_DB}" == "false" ]]; then + if [[ "${EXCLUDE_DB}" == "true" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -225,33 +194,29 @@ docker-arm64v8-build: export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" - # Run this job in a branch where a Dockerfile exists rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE docker-arm64v8-no-db-build: - # Use the official docker image. image: docker:latest stage: build variables: ARCH_TYPE: "arm64v8" - INCLUDE_DB: "false" + EXCLUDE_DB: "true" DOCKER_FILE: "Dockerfile.arm64v8" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - export BRANCH_PREFIX="" ; + export BRANCH_PREFIX="latest" ; export NO_DB_SUFIX="" ; export BRANCH_SEP="" ; @@ -259,7 +224,7 @@ docker-arm64v8-no-db-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi - if [[ "${INCLUDE_DB}" == "false" ]]; then + if [[ "${EXCLUDE_DB}" == "true" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -270,34 +235,29 @@ docker-arm64v8-no-db-build: export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" - # Run this job in a branch where a Dockerfile exists rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE - docker-nvidia-build: - # Use the official docker image. image: docker:latest stage: build variables: ARCH_TYPE: "nvidia" - INCLUDE_DB: "true" + EXCLUDE_DB: "false" DOCKER_FILE: "Dockerfile.nvidia" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - export BRANCH_PREFIX="" ; + export BRANCH_PREFIX="latest" ; export NO_DB_SUFIX="" ; export BRANCH_SEP="" ; @@ -305,7 +265,7 @@ docker-nvidia-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi - if [[ "${INCLUDE_DB}" == "false" ]]; then + if [[ "${EXCLUDE_DB}" == "true" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -316,33 +276,29 @@ docker-nvidia-build: export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" - # Run this job in a branch where a Dockerfile exists rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE docker-nvidia-no-db-build: - # Use the official docker image. image: docker:latest stage: build variables: ARCH_TYPE: "nvidia" - INCLUDE_DB: "false" + EXCLUDE_DB: "true" DOCKER_FILE: "Dockerfile.nvidia" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - export BRANCH_PREFIX="" ; + export BRANCH_PREFIX="latest" ; export NO_DB_SUFIX="" ; export BRANCH_SEP="" ; @@ -350,7 +306,7 @@ docker-nvidia-no-db-build: export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; fi - if [[ "${INCLUDE_DB}" == "false" ]]; then + if [[ "${EXCLUDE_DB}" == "true" ]]; then export NO_DB_SUFIX="-no-db" ; fi @@ -361,12 +317,11 @@ docker-nvidia-no-db-build: export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${INCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" - # Run this job in a branch where a Dockerfile exists rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE From e0201870759516db3054f5a1b237a5c92b4d5dda Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 09:42:30 +0300 Subject: [PATCH 024/112] allow arm64v8 always (test) --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05b8f29e..bac2652c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -197,8 +197,8 @@ docker-arm64v8-build: - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE @@ -238,8 +238,8 @@ docker-arm64v8-no-db-build: - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE From 5b2db64d59f012e7078198767525ab18d119ba6c Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Wed, 27 Sep 2023 09:43:25 +0300 Subject: [PATCH 025/112] remove test of arm64v8 --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bac2652c..05b8f29e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -197,8 +197,8 @@ docker-arm64v8-build: - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE @@ -238,8 +238,8 @@ docker-arm64v8-no-db-build: - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" - docker push "${IMAGE_NAME}" rules: - # - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE From 27ef2ce6f12c40b521648805ed9421af3bf4d321 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Thu, 28 Sep 2023 15:51:18 +0300 Subject: [PATCH 026/112] remove non-default dockerfile and injection of base image as part of the build process, adjust CI/CD process, clean up --- .gitlab-ci.yml | 230 ++++++++++----------------------- Docker/install_dependencies.sh | 86 ------------ Docker/install_ffmpeg.sh | 21 +++ Docker/install_mariadb.sh | 19 +++ Docker/install_nodejs.sh | 21 +++ Dockerfile | 53 +++++++- Dockerfile.arm32v7 | 36 ------ Dockerfile.arm64v8 | 36 ------ Dockerfile.nvidia | 39 ------ 9 files changed, 178 insertions(+), 363 deletions(-) delete mode 100644 Docker/install_dependencies.sh create mode 100644 Docker/install_ffmpeg.sh create mode 100644 Docker/install_mariadb.sh create mode 100644 Docker/install_nodejs.sh delete mode 100644 Dockerfile.arm32v7 delete mode 100644 Dockerfile.arm64v8 delete mode 100644 Dockerfile.nvidia diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05b8f29e..a4d216c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,35 +2,24 @@ docker-latest-build: image: docker:latest stage: build variables: - ARCH_TYPE: "" EXCLUDE_DB: "false" - DOCKER_FILE: "Dockerfile" + BASE_IMAGE: "node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - | - export BRANCH_PREFIX="latest" ; - export NO_DB_SUFIX="" ; - export BRANCH_SEP="" ; + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" - if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; - fi - - if [[ "${EXCLUDE_DB}" == "true" ]]; then - export NO_DB_SUFIX="-no-db" ; - fi - - if [[ "${ARCH_TYPE}" != "" ]]; then - export BRANCH_SEP="-" ; - fi - - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" @@ -43,285 +32,208 @@ docker-latest-no-db-build: image: docker:latest stage: build variables: - ARCH_TYPE: "" EXCLUDE_DB: "true" - DOCKER_FILE: "Dockerfile" + BASE_IMAGE: "node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - | - export BRANCH_PREFIX="latest" ; - export NO_DB_SUFIX="" ; - export BRANCH_SEP="" ; + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" - if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; - fi - - if [[ "${EXCLUDE_DB}" == "true" ]]; then - export NO_DB_SUFIX="-no-db" ; - fi - - if [[ "${ARCH_TYPE}" != "" ]]; then - export BRANCH_SEP="-" ; - fi - - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - $DOCKER_FILE + - Dockerfile docker-arm32v7-build: image: docker:latest stage: build variables: - ARCH_TYPE: "arm32v7" EXCLUDE_DB: "false" - DOCKER_FILE: "Dockerfile.arm32v7" + BASE_IMAGE: "arm32v7/node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - | - export BRANCH_PREFIX="latest" ; - export NO_DB_SUFIX="" ; - export BRANCH_SEP="" ; + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" - if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; - fi - - if [[ "${EXCLUDE_DB}" == "true" ]]; then - export NO_DB_SUFIX="-no-db" ; - fi - - if [[ "${ARCH_TYPE}" != "" ]]; then - export BRANCH_SEP="-" ; - fi - - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - $DOCKER_FILE + - Dockerfile docker-arm32v7-no-db-build: image: docker:latest stage: build variables: - ARCH_TYPE: "arm32v7" EXCLUDE_DB: "true" - DOCKER_FILE: "Dockerfile.arm32v7" + BASE_IMAGE: "arm32v7/node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - | - export BRANCH_PREFIX="latest" ; - export NO_DB_SUFIX="" ; - export BRANCH_SEP="" ; + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" - if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; - fi - - if [[ "${EXCLUDE_DB}" == "true" ]]; then - export NO_DB_SUFIX="-no-db" ; - fi - - if [[ "${ARCH_TYPE}" != "" ]]; then - export BRANCH_SEP="-" ; - fi - - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - $DOCKER_FILE + - Dockerfile docker-arm64v8-build: image: docker:latest stage: build variables: - ARCH_TYPE: "arm64v8" EXCLUDE_DB: "false" - DOCKER_FILE: "Dockerfile.arm64v8" + BASE_IMAGE: "arm64v8/node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - | - export BRANCH_PREFIX="latest" ; - export NO_DB_SUFIX="" ; - export BRANCH_SEP="" ; + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" - if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; - fi - - if [[ "${EXCLUDE_DB}" == "true" ]]; then - export NO_DB_SUFIX="-no-db" ; - fi - - if [[ "${ARCH_TYPE}" != "" ]]; then - export BRANCH_SEP="-" ; - fi - - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - $DOCKER_FILE + - Dockerfile docker-arm64v8-no-db-build: image: docker:latest stage: build variables: - ARCH_TYPE: "arm64v8" EXCLUDE_DB: "true" - DOCKER_FILE: "Dockerfile.arm64v8" + BASE_IMAGE: "arm64v8/node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - | - export BRANCH_PREFIX="latest" ; - export NO_DB_SUFIX="" ; - export BRANCH_SEP="" ; + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" - if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; - fi - - if [[ "${EXCLUDE_DB}" == "true" ]]; then - export NO_DB_SUFIX="-no-db" ; - fi - - if [[ "${ARCH_TYPE}" != "" ]]; then - export BRANCH_SEP="-" ; - fi - - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - $DOCKER_FILE + - Dockerfile docker-nvidia-build: image: docker:latest stage: build variables: - ARCH_TYPE: "nvidia" EXCLUDE_DB: "false" - DOCKER_FILE: "Dockerfile.nvidia" + BASE_IMAGE: "nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - | - export BRANCH_PREFIX="latest" ; - export NO_DB_SUFIX="" ; - export BRANCH_SEP="" ; + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" - if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; - fi - - if [[ "${EXCLUDE_DB}" == "true" ]]; then - export NO_DB_SUFIX="-no-db" ; - fi - - if [[ "${ARCH_TYPE}" != "" ]]; then - export BRANCH_SEP="-" ; - fi - - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - $DOCKER_FILE + - Dockerfile docker-nvidia-no-db-build: image: docker:latest stage: build variables: - ARCH_TYPE: "nvidia" EXCLUDE_DB: "true" - DOCKER_FILE: "Dockerfile.nvidia" + BASE_IMAGE: "nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - | - export BRANCH_PREFIX="latest" ; - export NO_DB_SUFIX="" ; - export BRANCH_SEP="" ; + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" - if [[ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]]; then - export BRANCH_PREFIX="${CI_COMMIT_REF_SLUG}" ; - fi - - if [[ "${EXCLUDE_DB}" == "true" ]]; then - export NO_DB_SUFIX="-no-db" ; - fi - - if [[ "${ARCH_TYPE}" != "" ]]; then - export BRANCH_SEP="-" ; - fi - - export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${BRANCH_SEP}${ARCH_TYPE}${NO_DB_SUFIX}" ; + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; - - docker build --pull --build-arg VAR_EXCLUDE_DB="${EXCLUDE_DB}" -t "${IMAGE_NAME}" . -f "${DOCKER_FILE}" + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - $DOCKER_FILE + - Dockerfile diff --git a/Docker/install_dependencies.sh b/Docker/install_dependencies.sh deleted file mode 100644 index b1622cc5..00000000 --- a/Docker/install_dependencies.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/sh -mkdir -p /etc/apt/keyrings - -apt update -y --fix-missing -apt upgrade -y -apt update -y --fix-missing - -apt install -y \ - wget \ - curl \ - net-tools \ - software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - tar \ - sudo \ - xz-utils \ - ca-certificates \ - gnupg \ - apt-utils - -if [ "$INSTALL_NODE" = "true" ] ; then set -ex; \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg ; \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list ; \ - apt update -y --fix-missing ; \ - apt upgrade -y ; \ - apt install nodejs -y ; fi - -node -v -npm -v - -npm i npm@latest pm2 pg -g --save -npm install --unsafe-perm - -chmod 777 /home/Shinobi -chmod -R 777 /home/Shinobi/plugins -chmod -f +x /home/Shinobi/Docker/init.sh - -sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh - -ffmpeg -version - -# Install MariaDB server... the debian way -if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - mkdir -p /var/lib/mysql; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; \ - sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi diff --git a/Docker/install_ffmpeg.sh b/Docker/install_ffmpeg.sh new file mode 100644 index 00000000..0732b99b --- /dev/null +++ b/Docker/install_ffmpeg.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +apt-get install -y \ + libfreetype6-dev \ + libgnutls28-dev \ + libmp3lame-dev \ + libass-dev \ + libogg-dev \ + libtheora-dev \ + libvorbis-dev \ + libvpx-dev \ + libwebp-dev \ + libssh2-1-dev \ + libopus-dev \ + librtmp-dev \ + libx264-dev \ + libx265-dev \ + x264 \ + ffmpeg + +ffmpeg -version \ No newline at end of file diff --git a/Docker/install_mariadb.sh b/Docker/install_mariadb.sh new file mode 100644 index 00000000..efb91341 --- /dev/null +++ b/Docker/install_mariadb.sh @@ -0,0 +1,19 @@ +#!/bin/sh +$EXCLUDE_DB > /dev/null + +if [ "$?" -ne 0 ] ; then + echo "Installing MariaDB" + set -ex + { \ + echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ + echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ + } | debconf-set-selections + + mkdir -p /var/lib/mysql + apt-get update + + apt-get install -y mariadb-server socat + + find /etc/mysql/ -name '*.cnf' -print0 | xargs -0 grep -lZE '^(bind-address|log)' | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/' + sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf +fi \ No newline at end of file diff --git a/Docker/install_nodejs.sh b/Docker/install_nodejs.sh new file mode 100644 index 00000000..7833b739 --- /dev/null +++ b/Docker/install_nodejs.sh @@ -0,0 +1,21 @@ +#!/bin/sh +which node > /dev/null + +if [ "$?" -ne 0 ] ; then + echo "Installing NodeJS 18" + mkdir -p /etc/apt/keyrings + apt-get install -y ca-certificates gnupg + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + + apt-get update -y --fix-missing + apt-get upgrade -y + apt-get install nodejs -y +fi + +node -v +npm -v + +npm i npm@latest pm2 pg -g --save +npm install --unsafe-perm diff --git a/Dockerfile b/Dockerfile index 4414a790..6157c72d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ -FROM node:18-bullseye-slim - +ARG BASE_IMAGE=node:18-bullseye-slim +FROM ${BASE_IMAGE} + ARG DEBIAN_FRONTEND=noninteractive \ - VAR_EXCLUDE_DB=false + EXCLUDE_DB=false ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -18,12 +19,50 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB + DB_DISABLE_INCLUDED=$EXCLUDE_DB WORKDIR /home/Shinobi COPY . ./ -RUN sh /home/Shinobi/Docker/install_dependencies.sh +RUN apt-get update -y +RUN apt-get upgrade -y + +RUN apt-get install -y \ + wget \ + curl \ + net-tools \ + software-properties-common \ + build-essential \ + git \ + python3 \ + sudo \ + pkg-config \ + apt-utils \ + yasm \ + bzip2 \ + coreutils \ + procps \ + gnutls-bin \ + nasm \ + tar \ + make \ + g++ \ + gcc \ + tar \ + xz-utils + +RUN sh /home/Shinobi/Docker/install_ffmpeg.sh +RUN sh /home/Shinobi/Docker/install_mariadb.sh +RUN sh /home/Shinobi/Docker/install_nodejs.sh + +RUN chmod 777 /home/Shinobi +RUN chmod -R 777 /home/Shinobi/plugins +RUN chmod -f +x /home/Shinobi/Docker/init.sh + +RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh + +RUN apt-get update -y --fix-missing +RUN apt-get upgrade -y VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] @@ -31,6 +70,6 @@ VOLUME ["/config"] EXPOSE 8080 443 21 25 -ENTRYPOINT ["sh", "/home/Shinobi/Docker/init.sh"] +ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] +CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] \ No newline at end of file diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 deleted file mode 100644 index 9957f392..00000000 --- a/Dockerfile.arm32v7 +++ /dev/null @@ -1,36 +0,0 @@ -FROM arm32v7/node:18-bullseye-slim - -ARG DEBIAN_FRONTEND=noninteractive \ - VAR_EXCLUDE_DB=false - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB - -WORKDIR /home/Shinobi -COPY . ./ - -RUN sh /home/Shinobi/Docker/install_dependencies.sh - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 deleted file mode 100644 index 66b51fa2..00000000 --- a/Dockerfile.arm64v8 +++ /dev/null @@ -1,36 +0,0 @@ -FROM arm64v8/node:18-bullseye-slim - -ARG DEBIAN_FRONTEND=noninteractive \ - VAR_EXCLUDE_DB=false - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB - -WORKDIR /home/Shinobi -COPY . ./ - -RUN sh /home/Shinobi/Docker/install_dependencies.sh - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia deleted file mode 100644 index 886b92b9..00000000 --- a/Dockerfile.nvidia +++ /dev/null @@ -1,39 +0,0 @@ -FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04 - -ARG DEBIAN_FRONTEND=noninteractive \ - VAR_EXCLUDE_DB=false \ - VAR_INSTALL_NODE=true - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB \ - NODE_MAJOR=18 \ - INSTALL_NODE=$VAR_INSTALL_NODE - -WORKDIR /home/Shinobi -COPY . ./ - -RUN sh /home/Shinobi/Docker/install_dependencies.sh - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["sh", "/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] From d46c25a0d065022beed53fcbacb353b1121266d2 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Fri, 29 Sep 2023 13:14:44 +0000 Subject: [PATCH 027/112] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4d216c2..0a679cb6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,8 +22,8 @@ docker-latest-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - $DOCKER_FILE From 74c168eec07b027e09990942aa5efd7acfa50ade Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Fri, 29 Sep 2023 13:15:48 +0000 Subject: [PATCH 028/112] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a679cb6..0e7481fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,7 +26,7 @@ docker-latest-build: # when: never - if: $CI_COMMIT_BRANCH exists: - - $DOCKER_FILE + - Dockerfile docker-latest-no-db-build: image: docker:latest From 93a10b9f8dc36bf14fea8e703cadbae1ea330e17 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Fri, 29 Sep 2023 13:22:04 +0000 Subject: [PATCH 029/112] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0e7481fa..4a23804d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,8 +22,8 @@ docker-latest-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile @@ -52,8 +52,8 @@ docker-latest-no-db-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile From 29697d0966fd63e86a8f0cffaebc625728a86893 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Fri, 29 Sep 2023 13:27:16 +0000 Subject: [PATCH 030/112] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a23804d..ca85613c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,8 +52,8 @@ docker-latest-no-db-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile @@ -112,8 +112,8 @@ docker-arm32v7-no-db-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile @@ -172,8 +172,8 @@ docker-arm64v8-no-db-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile @@ -232,8 +232,8 @@ docker-nvidia-no-db-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - when: never + #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + # when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile From 609218c86857f2f451efac298ccccfce1b3195f2 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Fri, 29 Sep 2023 14:16:33 +0000 Subject: [PATCH 031/112] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca85613c..97d8f284 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -112,8 +112,8 @@ docker-arm32v7-no-db-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile @@ -172,8 +172,8 @@ docker-arm64v8-no-db-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile @@ -232,8 +232,8 @@ docker-nvidia-no-db-build: - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" - docker push "${IMAGE_NAME}" rules: - #- if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" - # when: never + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never - if: $CI_COMMIT_BRANCH exists: - Dockerfile From 8b9977fbf83d853ad0b8314ff565fe0d11b22789 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 3 Oct 2023 19:29:31 -0700 Subject: [PATCH 032/112] add global webhook method selector --- libs/notifications/webhook.js | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/libs/notifications/webhook.js b/libs/notifications/webhook.js index 194546c3..b133deef 100644 --- a/libs/notifications/webhook.js +++ b/libs/notifications/webhook.js @@ -240,7 +240,29 @@ module.exports = function(s,config,lang,getSnapshot){ "placeholder": "http://your-webhook-point/onEvent/{{INNER_EVENT_TITLE}}?info={{INNER_EVENT_INFO}}", "field": lang["Webhook URL"], "form-group-class":"u_global_webhook_input u_global_webhook_1", - } + }, + { + hidden: true, + "name": "detail=global_webhook_method", + "field": lang['Call Method'], + "default": "GET", + "form-group-class":"u_global_webhook_input u_global_webhook_1", + "fieldType": "select", + "possible": [ + { + "name": `GET (${lang.Default})`, + "value": "GET" + }, + { + "name": "PUT", + "value": "PUT" + }, + { + "name": "POST", + "value": "POST" + } + ] + }, ] } s.definitions["Event Filters"].blocks["Action for Selected"].info.push({ From 85126f0b9c72132e0910597ad5d5a601f10af073 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 4 Oct 2023 12:15:30 -0700 Subject: [PATCH 033/112] fix event time for titles --- libs/events/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/events/utils.js b/libs/events/utils.js index f366fb8d..004d6950 100644 --- a/libs/events/utils.js +++ b/libs/events/utils.js @@ -448,8 +448,8 @@ module.exports = (s,config,lang) => { createEventBasedRecording(d,moment(eventTime).subtract(secondBefore,'seconds').format('YYYY-MM-DDTHH-mm-ss')) } d.currentTime = eventTime - d.currentTimestamp = s.timeObject(d.currentTime).format() - d.screenshotName = eventDetails.reason + '_'+(monitorConfig.name.replace(/[^\w\s]/gi,''))+'_'+d.id+'_'+d.ke+'_'+s.formattedTime() + d.currentTimestamp = s.timeObject(eventTime).format() + d.screenshotName = eventDetails.reason + '_'+(monitorConfig.name.replace(/[^\w\s]/gi,''))+'_'+d.id+'_'+d.ke+'_'+s.formattedTime(eventTime) d.screenshotBuffer = null if(filter.webhook && monitorDetails.detector_webhook === '1' && !s.group[d.ke].activeMonitors[d.id].detector_webhook){ From 830d702241bf3b39f24abe38ecbbbda4c01da17c Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 4 Oct 2023 15:38:16 -0700 Subject: [PATCH 034/112] Update dataPort.js --- libs/dataPort.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/dataPort.js b/libs/dataPort.js index a20c5143..3854826e 100644 --- a/libs/dataPort.js +++ b/libs/dataPort.js @@ -39,6 +39,8 @@ module.exports = function(s,config,lang,app,io){ s.debugLog(data.data) break; default: + console.log(`No Data Port Handler!`) + console.log(`here's what we got :`) console.log(data) break; } From e1e35344fed9e939b17d8202a8a62060f207d4d0 Mon Sep 17 00:00:00 2001 From: Moe Date: Sat, 7 Oct 2023 10:01:47 -0700 Subject: [PATCH 035/112] fix "Send Frames" toggle for object when motion off --- libs/ffmpeg/builders.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/ffmpeg/builders.js b/libs/ffmpeg/builders.js index faa52386..eca1fe1c 100644 --- a/libs/ffmpeg/builders.js +++ b/libs/ffmpeg/builders.js @@ -617,7 +617,8 @@ module.exports = (s,config,lang) => { const sendFramesGlobally = (e.details.detector_send_frames === '1') const objectDetectorOutputIsEnabled = (e.details.detector_use_detect_object === '1') const builtInMotionDetectorIsEnabled = (e.details.detector_pam === '1') - const sendFramesToObjectDetector = (e.details.detector_send_frames_object !== '0' && e.details.detector_use_detect_object === '1') + const objectDetectorSendFrames = e.details.detector_send_frames_object !== '0' || e.details.detector_pam !== '1' + const sendFramesToObjectDetector = (objectDetectorSendFrames && e.details.detector_use_detect_object === '1') const baseWidth = e.details.detector_scale_x ? e.details.detector_scale_x : '640' const baseHeight = e.details.detector_scale_y ? e.details.detector_scale_y : '480' const baseDimensionsFlag = `-s ${baseWidth}x${baseHeight}` From 48d564e549597929f309adb8ed25f7c761f3fb78 Mon Sep 17 00:00:00 2001 From: chen moye Date: Fri, 13 Oct 2023 01:48:21 +0000 Subject: [PATCH 036/112] fix downloadTimelapseVideo --- web/assets/js/bs5.timelapseViewer.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/assets/js/bs5.timelapseViewer.js b/web/assets/js/bs5.timelapseViewer.js index a2271054..cc6b00c8 100644 --- a/web/assets/js/bs5.timelapseViewer.js +++ b/web/assets/js/bs5.timelapseViewer.js @@ -311,6 +311,13 @@ $(document).ready(function(e){ function downloadTimelapseFrame(frame){ downloadFile(frame.href,frame.filename) } + function buildFileBinUrl(data){ + return apiBaseUrl + '/fileBin/' + data.ke + '/' + data.mid + '/' + data.name + } + function downloadTimelapseVideo(data){ + var downloadUrl = buildFileBinUrl(data) + downloadFile(downloadUrl,data.name) + } function onTimelapseVideoBuildComplete(data){ var saveBuiltVideo = dashboardOptions().switches.timelapseSaveBuiltVideo if(saveBuiltVideo === 1){ From 4d71c6836d75f2f100354688700fac82c6b9a67c Mon Sep 17 00:00:00 2001 From: Moe Date: Thu, 19 Oct 2023 15:32:38 +0000 Subject: [PATCH 037/112] Additional Density - Performance Upgrades and Bug Fixes --- .gitlab-ci.yml | 246 +++++++++++++++++++++------ Docker/install_ffmpeg.sh | 21 +++ Docker/install_mariadb.sh | 19 +++ Docker/install_nodejs.sh | 21 +++ Dockerfile | 108 ++++-------- Dockerfile.arm32v7 | 109 ------------ Dockerfile.nodb | 97 ----------- Dockerfile.nvidia | 118 ------------- definitions/base.js | 3 +- languages/en_CA.json | 2 + libs/dataPort.js | 2 + libs/events/utils.js | 7 +- libs/extenders.js | 3 + libs/ffmpeg/builders.js | 5 +- libs/ffmpeg/possibleWarnings.js | 7 +- libs/ffmpeg/utils.js | 13 ++ libs/monitor.js | 13 +- libs/monitor/utils.js | 30 ++-- libs/notifications/webhook.js | 24 ++- libs/uploaders/amazonS3.js | 190 +++++++++++++-------- libs/uploaders/backblazeB2.js | 52 +++--- libs/uploaders/s3based.js | 200 +++++++++++++--------- libs/user.js | 1 + libs/user/logger.js | 31 ++++ libs/webServerPaths.js | 25 ++- libs/webServerStreamPaths.js | 2 +- web/assets/css/dashboard.css | 5 + web/assets/js/bs5.dashboard-base.js | 10 +- web/assets/js/bs5.startup.js | 2 +- web/assets/js/bs5.timelapseViewer.js | 7 + web/assets/js/bs5.timeline.js | 23 ++- web/assets/js/bs5.videosTable.js | 1 + 32 files changed, 731 insertions(+), 666 deletions(-) create mode 100644 Docker/install_ffmpeg.sh create mode 100644 Docker/install_mariadb.sh create mode 100644 Docker/install_nodejs.sh delete mode 100644 Dockerfile.arm32v7 delete mode 100644 Dockerfile.nodb delete mode 100644 Dockerfile.nvidia create mode 100644 libs/user/logger.js diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79d095a1..97d8f284 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,37 +1,56 @@ -# This file is a template, and might need editing before it works on your project. -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml - -# Build a Docker image with CI/CD and push to the GitLab registry. -# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html -# -# This template uses one generic job with conditional builds -# for the default branch and all other (MR) branches. - docker-latest-build: - # Use the official docker image. image: docker:latest stage: build + variables: + EXCLUDE_DB: "false" + BASE_IMAGE: "node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag="" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" - else - tag=":$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" - fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . - - docker push "$CI_REGISTRY_IMAGE${tag}" - # Run this job in a branch where a Dockerfile exists + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + +docker-latest-no-db-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "true" + BASE_IMAGE: "node:18-bullseye-slim" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never @@ -40,60 +59,181 @@ docker-latest-build: - Dockerfile docker-arm32v7-build: - # Use the official docker image. image: docker:latest stage: build + variables: + EXCLUDE_DB: "false" + BASE_IMAGE: "arm32v7/node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":arm32v7" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm32v7'" - else - tag=":arm32v7-$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" - fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm32v7 - - docker push "$CI_REGISTRY_IMAGE${tag}" - # Run this job in a branch where a Dockerfile exists + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.arm32v7 + - Dockerfile +docker-arm32v7-no-db-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "true" + BASE_IMAGE: "arm32v7/node:18-bullseye-slim" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + +docker-arm64v8-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "false" + BASE_IMAGE: "arm64v8/node:18-bullseye-slim" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + +docker-arm64v8-no-db-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "true" + BASE_IMAGE: "arm64v8/node:18-bullseye-slim" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile docker-nvidia-build: - # Use the official docker image. image: docker:latest stage: build + variables: + EXCLUDE_DB: "false" + BASE_IMAGE: "nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":nvidia" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'nvidia'" - else - tag=":nvidia-$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" - fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia - - docker push "$CI_REGISTRY_IMAGE${tag}" - # Run this job in a branch where a Dockerfile exists + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.nvidia + - Dockerfile + +docker-nvidia-no-db-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "true" + BASE_IMAGE: "nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile diff --git a/Docker/install_ffmpeg.sh b/Docker/install_ffmpeg.sh new file mode 100644 index 00000000..0732b99b --- /dev/null +++ b/Docker/install_ffmpeg.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +apt-get install -y \ + libfreetype6-dev \ + libgnutls28-dev \ + libmp3lame-dev \ + libass-dev \ + libogg-dev \ + libtheora-dev \ + libvorbis-dev \ + libvpx-dev \ + libwebp-dev \ + libssh2-1-dev \ + libopus-dev \ + librtmp-dev \ + libx264-dev \ + libx265-dev \ + x264 \ + ffmpeg + +ffmpeg -version \ No newline at end of file diff --git a/Docker/install_mariadb.sh b/Docker/install_mariadb.sh new file mode 100644 index 00000000..efb91341 --- /dev/null +++ b/Docker/install_mariadb.sh @@ -0,0 +1,19 @@ +#!/bin/sh +$EXCLUDE_DB > /dev/null + +if [ "$?" -ne 0 ] ; then + echo "Installing MariaDB" + set -ex + { \ + echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ + echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ + } | debconf-set-selections + + mkdir -p /var/lib/mysql + apt-get update + + apt-get install -y mariadb-server socat + + find /etc/mysql/ -name '*.cnf' -print0 | xargs -0 grep -lZE '^(bind-address|log)' | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/' + sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf +fi \ No newline at end of file diff --git a/Docker/install_nodejs.sh b/Docker/install_nodejs.sh new file mode 100644 index 00000000..7833b739 --- /dev/null +++ b/Docker/install_nodejs.sh @@ -0,0 +1,21 @@ +#!/bin/sh +which node > /dev/null + +if [ "$?" -ne 0 ] ; then + echo "Installing NodeJS 18" + mkdir -p /etc/apt/keyrings + apt-get install -y ca-certificates gnupg + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + + apt-get update -y --fix-missing + apt-get upgrade -y + apt-get install nodejs -y +fi + +node -v +npm -v + +npm i npm@latest pm2 pg -g --save +npm install --unsafe-perm diff --git a/Dockerfile b/Dockerfile index a94abea0..6157c72d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,8 @@ -FROM node:16-buster-slim +ARG BASE_IMAGE=node:18-bullseye-slim +FROM ${BASE_IMAGE} + +ARG DEBIAN_FRONTEND=noninteractive \ + EXCLUDE_DB=false ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -15,97 +19,57 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive + DB_DISABLE_INCLUDED=$EXCLUDE_DB -RUN mkdir -p /home/Shinobi /config /var/lib/mysql +WORKDIR /home/Shinobi +COPY . ./ -RUN apt update -y -RUN apt install wget curl net-tools -y +RUN apt-get update -y +RUN apt-get upgrade -y -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ +RUN apt-get install -y \ + wget \ + curl \ + net-tools \ + software-properties-common \ build-essential \ + git \ + python3 \ + sudo \ + pkg-config \ + apt-utils \ + yasm \ bzip2 \ coreutils \ procps \ gnutls-bin \ nasm \ tar \ - x264 + make \ + g++ \ + gcc \ + tar \ + xz-utils -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils +RUN sh /home/Shinobi/Docker/install_ffmpeg.sh +RUN sh /home/Shinobi/Docker/install_mariadb.sh +RUN sh /home/Shinobi/Docker/install_nodejs.sh - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins RUN chmod 777 /home/Shinobi -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ - -# Copy default configuration files +RUN chmod -R 777 /home/Shinobi/plugins RUN chmod -f +x /home/Shinobi/Docker/init.sh + RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi + +RUN apt-get update -y --fix-missing +RUN apt-get upgrade -y VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] EXPOSE 8080 443 21 25 -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] +ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] +CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] \ No newline at end of file diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 deleted file mode 100644 index 2e724f77..00000000 --- a/Dockerfile.arm32v7 +++ /dev/null @@ -1,109 +0,0 @@ -FROM arm32v7/node:16-buster-slim - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm && \ - apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y zip - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install pm2 -g && \ - npm install --unsafe-perm -COPY ./Docker/pm2.yml ./ - -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] - -EXPOSE 8080 - -ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "pm2.yml" ] diff --git a/Dockerfile.nodb b/Dockerfile.nodb deleted file mode 100644 index 8648484f..00000000 --- a/Dockerfile.nodb +++ /dev/null @@ -1,97 +0,0 @@ -FROM node:16-buster-slim - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=true -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y zip - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ - -RUN npm i pg --save - -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia deleted file mode 100644 index d6221971..00000000 --- a/Dockerfile.nvidia +++ /dev/null @@ -1,118 +0,0 @@ -FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu20.04 - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Additional packages to support NodeJS as the base build is NVidia CUDA -RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - -RUN apt install nodejs -y - -RUN node -v -RUN npm -v - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ - -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/definitions/base.js b/definitions/base.js index 80f6498b..136b50b1 100644 --- a/definitions/base.js +++ b/definitions/base.js @@ -748,8 +748,7 @@ module.exports = function(s,config,lang){ "name": "detail=stream_type", "field": lang["Stream Type"], "description": lang["fieldTextStreamType"], - "default": "mp4", - "example": "", + "default": "hls", "selector": "h_st", "fieldType": "select", "attribute": `triggerChange="#add_monitor [detail=stream_vcodec]" triggerChangeIgnore="b64,mjpeg,jpeg,gif"`, diff --git a/languages/en_CA.json b/languages/en_CA.json index f4b4e075..e1923e31 100644 --- a/languages/en_CA.json +++ b/languages/en_CA.json @@ -145,6 +145,7 @@ "Input Settings": "Input Settings", "Connection": "Connection", "Video Set": "Video Set", + "Video Accessed": "Video Accessed", "notEnoughFramesText1": "Not Enough Frames for compilation.", "Allow API Trigger": "Allow API Trigger", "When Detector is Off": "When Detector is Off", @@ -480,6 +481,7 @@ "Database": "Database", "Database Not Found": "Database Not Found", "User Not Found": "User Not Found", + "User Logged Out": "User Logged Out", "Not Found": "Not Found", "Save Links to Database": "Save Links to Database", "Uploaders": "Uploaders", diff --git a/libs/dataPort.js b/libs/dataPort.js index a20c5143..3854826e 100644 --- a/libs/dataPort.js +++ b/libs/dataPort.js @@ -39,6 +39,8 @@ module.exports = function(s,config,lang,app,io){ s.debugLog(data.data) break; default: + console.log(`No Data Port Handler!`) + console.log(`here's what we got :`) console.log(data) break; } diff --git a/libs/events/utils.js b/libs/events/utils.js index 98e50cdb..004d6950 100644 --- a/libs/events/utils.js +++ b/libs/events/utils.js @@ -448,8 +448,8 @@ module.exports = (s,config,lang) => { createEventBasedRecording(d,moment(eventTime).subtract(secondBefore,'seconds').format('YYYY-MM-DDTHH-mm-ss')) } d.currentTime = eventTime - d.currentTimestamp = s.timeObject(d.currentTime).format() - d.screenshotName = eventDetails.reason + '_'+(monitorConfig.name.replace(/[^\w\s]/gi,''))+'_'+d.id+'_'+d.ke+'_'+s.formattedTime() + d.currentTimestamp = s.timeObject(eventTime).format() + d.screenshotName = eventDetails.reason + '_'+(monitorConfig.name.replace(/[^\w\s]/gi,''))+'_'+d.id+'_'+d.ke+'_'+s.formattedTime(eventTime) d.screenshotBuffer = null if(filter.webhook && monitorDetails.detector_webhook === '1' && !s.group[d.ke].activeMonitors[d.id].detector_webhook){ @@ -836,10 +836,11 @@ module.exports = (s,config,lang) => { function getObjectTagsFromMatrices(d){ if(d.details.reason === 'motion'){ return [getTagWithIcon(lang.Motion)] - }else{ + }else if(d.details.matrices){ const matrices = d.details.matrices return [...new Set(matrices.map(matrix => getTagWithIcon(matrix.tag)))]; } + return [getTagWithIcon(d.details.reason)] } function getObjectTagNotifyText(d){ const monitorId = d.mid || d.id diff --git a/libs/extenders.js b/libs/extenders.js index 61b66361..7cf3595d 100644 --- a/libs/extenders.js +++ b/libs/extenders.js @@ -25,6 +25,8 @@ module.exports = function(s,config){ createExtension(`beforeAccountSave`) createExtension(`onTwoFactorAuthCodeNotification`) createExtension(`onStalePurgeLock`) + createExtension(`onVideoAccess`) + createExtension(`onLogout`) ////// EVENTS ////// createExtension(`onEventTrigger`) createExtension(`onEventTriggerBeforeFilter`) @@ -62,6 +64,7 @@ module.exports = function(s,config){ createExtension(`onEventBasedRecordingComplete`) createExtension(`onEventBasedRecordingStart`) createExtension(`onBeforeInsertCompletedVideo`) + createExtension(`onCloudVideoUploaded`) /////// TIMELAPSE //////// createExtension(`onInsertTimelapseFrame`) } diff --git a/libs/ffmpeg/builders.js b/libs/ffmpeg/builders.js index 727ee359..eca1fe1c 100644 --- a/libs/ffmpeg/builders.js +++ b/libs/ffmpeg/builders.js @@ -548,7 +548,7 @@ module.exports = (s,config,lang) => { recordingFilters.push(`fps=${videoFps}`) } } - if(videoExtIsMp4){ + if(videoExtIsMp4 && !config.noDefaultRecordingSegmentFormatOptions){ customRecordingFlags.push(`-segment_format_options movflags=faststart`) } if(videoCodec === 'h264_vaapi'){ @@ -617,7 +617,8 @@ module.exports = (s,config,lang) => { const sendFramesGlobally = (e.details.detector_send_frames === '1') const objectDetectorOutputIsEnabled = (e.details.detector_use_detect_object === '1') const builtInMotionDetectorIsEnabled = (e.details.detector_pam === '1') - const sendFramesToObjectDetector = (e.details.detector_send_frames_object !== '0' && e.details.detector_use_detect_object === '1') + const objectDetectorSendFrames = e.details.detector_send_frames_object !== '0' || e.details.detector_pam !== '1' + const sendFramesToObjectDetector = (objectDetectorSendFrames && e.details.detector_use_detect_object === '1') const baseWidth = e.details.detector_scale_x ? e.details.detector_scale_x : '640' const baseHeight = e.details.detector_scale_y ? e.details.detector_scale_y : '480' const baseDimensionsFlag = `-s ${baseWidth}x${baseHeight}` diff --git a/libs/ffmpeg/possibleWarnings.js b/libs/ffmpeg/possibleWarnings.js index 8f4097bc..d271743b 100644 --- a/libs/ffmpeg/possibleWarnings.js +++ b/libs/ffmpeg/possibleWarnings.js @@ -1,5 +1,6 @@ module.exports = (monitor,probeResult,config,lang) => { const primaryVideoStream = probeResult.video[0] + const noPortOr554 = !monitor.port || monitor.port === 554 return [ { isTrue: monitor.details.stream_vcodec === 'copy' && primaryVideoStream.codec === 'hevc', @@ -83,7 +84,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && monitor.protocol === 'rtmp' ), title: lang['Automatic Field Fill'], @@ -95,7 +96,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && monitor.protocol === 'http' ), title: lang['Automatic Field Fill'], @@ -107,7 +108,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && (monitor.protocol === 'rtmps' || monitor.protocol === 'https') ), diff --git a/libs/ffmpeg/utils.js b/libs/ffmpeg/utils.js index e20402ec..e678051b 100644 --- a/libs/ffmpeg/utils.js +++ b/libs/ffmpeg/utils.js @@ -367,6 +367,18 @@ Run "npm install ffbinaries" to get this static FFmpeg downloader.` } return response } + async function getWarningChangesForMonitor(monitorConfig){ + const probeResponse = await probeMonitor(monitorConfig,2000,true) + const probeStreams = getStreamInfoFromProbe(probeResponse.result) + const warnings = createWarningsForConfiguration(monitorConfig,probeStreams) + const configPartial = warnings.length > 0 ? buildMonitorConfigPartialFromWarnings(warnings) : {} + return { + configPartial, + warnings, + probeResponse, + probeStreams, + } + } return { ffprobe: runFFprobe, probeMonitor: probeMonitor, @@ -386,5 +398,6 @@ Run "npm install ffbinaries" to get this static FFmpeg downloader.` checkStaticBuilds: checkStaticBuilds, checkVersion: checkVersion, checkHwAccelMethods: checkHwAccelMethods, + getWarningChangesForMonitor, } } diff --git a/libs/monitor.js b/libs/monitor.js index 4a366722..95506de6 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -13,6 +13,8 @@ module.exports = function(s,config,lang){ } = require('./basic/utils.js')(process.cwd(),config) const { splitForFFMPEG, + applyPartialToConfiguration, + getWarningChangesForMonitor, } = require('./ffmpeg/utils.js')(s,config,lang) const { processKill, @@ -551,7 +553,6 @@ module.exports = function(s,config,lang){ return } form.mid = form.mid.replace(/[^\w\s]/gi,'').replace(/ /g,'') - form = s.cleanMonitorObjectForDatabase(form) const selectResponse = await s.knexQueryPromise({ action: "select", columns: "*", @@ -570,6 +571,16 @@ module.exports = function(s,config,lang){ ke: form.ke, mon: form } + // auto correct + const { + configPartial, + warnings, + probeResponse, + probeStreams, + } = await getWarningChangesForMonitor(form) + applyPartialToConfiguration(form,configPartial) + form = s.cleanMonitorObjectForDatabase(form) + // if(monitorExists){ txData.new = false Object.keys(form).forEach(function(v){ diff --git a/libs/monitor/utils.js b/libs/monitor/utils.js index 5db663da..00cd7ddf 100644 --- a/libs/monitor/utils.js +++ b/libs/monitor/utils.js @@ -10,11 +10,8 @@ const SoundDetection = require('shinobi-sound-detection') const streamViewerCountTimeouts = {} module.exports = (s,config,lang) => { const { - probeMonitor, - getStreamInfoFromProbe, applyPartialToConfiguration, - createWarningsForConfiguration, - buildMonitorConfigPartialFromWarnings, + getWarningChangesForMonitor, createPipeArray, splitForFFMPEG, sanitizedFfmpegCommand, @@ -1353,6 +1350,7 @@ module.exports = (s,config,lang) => { // doFatalErrorCatch(e,d) // },15000) break; + case checkLog(d,'Could not find codec parameters'): case checkLog(d,'No route to host'): activeMonitor.timeoutToRestart = setTimeout(async () => { doFatalErrorCatch(e,d) @@ -1662,17 +1660,18 @@ module.exports = (s,config,lang) => { return; } if(config.probeMonitorOnStart === true){ - const probeResponse = await probeMonitor(monitorConfig,2000,true) - const probeStreams = getStreamInfoFromProbe(probeResponse.result) - activeMonitor.probeResult = probeStreams - const warnings = createWarningsForConfiguration(monitorConfig,probeStreams) - activeMonitor.warnings = warnings + const { + configPartial, + warnings, + probeResponse, + probeStreams, + } = await getWarningChangesForMonitor(monitorConfig) if(warnings.length > 0){ - const configPartial = buildMonitorConfigPartialFromWarnings(warnings) applyPartialToConfiguration(e,configPartial) applyPartialToConfiguration(activeMonitor,configPartial) applyPartialToConfiguration(s.group[groupKey].rawMonitorConfigurations[monitorId],configPartial) } + activeMonitor.warnings = warnings } activeMonitor.isStarted = true if(e.details && e.details.dir && e.details.dir !== ''){ @@ -1694,6 +1693,7 @@ module.exports = (s,config,lang) => { } try{ await launchMonitorProcesses(e) + resetStreamCheck(e) }catch(err){ console.error(err) } @@ -1748,9 +1748,13 @@ module.exports = (s,config,lang) => { } function isGroupBelowMaxMonitorCount(groupKey){ const theGroup = s.group[groupKey]; - const initData = theGroup.init; - const maxCamerasAllowed = parseInt(initData.max_camera) || false; - return (!maxCamerasAllowed || Object.keys(theGroup.activeMonitors).length <= parseInt(maxCamerasAllowed)) + try{ + const initData = theGroup.init; + const maxCamerasAllowed = parseInt(initData.max_camera) || false; + return (!maxCamerasAllowed || Object.keys(theGroup.activeMonitors).length <= parseInt(maxCamerasAllowed)) + }catch(err){ + return true + } } function getStreamDirectory(options){ const streamDir = s.dir.streams + options.ke + '/' + options.mid + '/' diff --git a/libs/notifications/webhook.js b/libs/notifications/webhook.js index 194546c3..b133deef 100644 --- a/libs/notifications/webhook.js +++ b/libs/notifications/webhook.js @@ -240,7 +240,29 @@ module.exports = function(s,config,lang,getSnapshot){ "placeholder": "http://your-webhook-point/onEvent/{{INNER_EVENT_TITLE}}?info={{INNER_EVENT_INFO}}", "field": lang["Webhook URL"], "form-group-class":"u_global_webhook_input u_global_webhook_1", - } + }, + { + hidden: true, + "name": "detail=global_webhook_method", + "field": lang['Call Method'], + "default": "GET", + "form-group-class":"u_global_webhook_input u_global_webhook_1", + "fieldType": "select", + "possible": [ + { + "name": `GET (${lang.Default})`, + "value": "GET" + }, + { + "name": "PUT", + "value": "PUT" + }, + { + "name": "POST", + "value": "POST" + } + ] + }, ] } s.definitions["Event Filters"].blocks["Action for Selected"].info.push({ diff --git a/libs/uploaders/amazonS3.js b/libs/uploaders/amazonS3.js index c9f02149..95dd6497 100644 --- a/libs/uploaders/amazonS3.js +++ b/libs/uploaders/amazonS3.js @@ -315,79 +315,123 @@ module.exports = function(s,config,lang){ "default": "", "example": "", "possible": [ - { - "name": "US West (N. California)", - "value": "us-west-1" - }, - { - "name": "US West (Oregon)", - "value": "us-west-2" - }, - { - "name": "US East (Ohio)", - "value": "us-east-2" - }, - { - "name": "US East (N. Virginia)", - "value": "us-east-1" - }, - { - "name": "Asia Pacific (Mumbai)", - "value": "ap-south-1" - }, - { - "name": "Asia Pacific (Seoul)", - "value": "ap-northeast-2" - }, - { - "name": "Asia Pacific (Osaka-Local)**", - "value": "ap-northeast-3" - }, - { - "name": "Asia Pacific (Singapore)", - "value": "ap-southeast-1" - }, - { - "name": "Asia Pacific (Sydney)", - "value": "ap-southeast-2" - }, - { - "name": "Asia Pacific (Tokyo)", - "value": "ap-northeast-1" - }, - { - "name": "Canada (Central)", - "value": "ca-central-1" - }, - { - "name": "China (Beijing)", - "value": "cn-north-1" - }, - { - "name": "China (Ningxia)", - "value": "cn-northwest-1" - }, - { - "name": "EU (Frankfurt)", - "value": "eu-central-1" - }, - { - "name": "EU (Ireland)", - "value": "eu-west-1" - }, - { - "name": "EU (London)", - "value": "eu-west-2" - }, - { - "name": "EU (Paris)", - "value": "eu-west-3" - }, - { - "name": "South America (São Paulo)", - "value": "sa-east-1" - } - ] + { + "name": "US West (N. California)", + "value": "us-west-1" + }, + { + "name": "US West (Oregon)", + "value": "us-west-2" + }, + { + "name": "US East (Ohio)", + "value": "us-east-2" + }, + { + "name": "US East (N. Virginia)", + "value": "us-east-1" + }, + { + "name": "Canada (Central)", + "value": "ca-central-1" + }, + { + "name": "South America (São Paulo)", + "value": "sa-east-1" + }, + { + "name": "EU (Frankfurt)", + "value": "eu-central-1" + }, + { + "name": "EU (Ireland)", + "value": "eu-west-1" + }, + { + "name": "EU (London)", + "value": "eu-west-2" + }, + { + "name": "EU (Paris)", + "value": "eu-west-3" + }, + { + "name": "Europe (Milan)", + "value": "eu-south-1" + }, + { + "name": "Europe (Spain)", + "value": "eu-south-2" + }, + { + "name": "Europe (Zurich)", + "value": "eu-central-2" + }, + { + "name": "Asia Pacific (Mumbai)", + "value": "ap-south-1" + }, + { + "name": "Asia Pacific (Seoul)", + "value": "ap-northeast-2" + }, + { + "name": "Asia Pacific (Osaka-Local)**", + "value": "ap-northeast-3" + }, + { + "name": "Asia Pacific (Singapore)", + "value": "ap-southeast-1" + }, + { + "name": "Asia Pacific (Sydney)", + "value": "ap-southeast-2" + }, + { + "name": "Asia Pacific (Tokyo)", + "value": "ap-northeast-1" + }, + { + "name": "Asia Pacific (Hong Kong)", + "value": "ap-east-1" + }, + { + "name": "Asia Pacific (Hyderabad)", + "value": "ap-south-2" + }, + { + "name": "Asia Pacific (Jakarta)", + "value": "ap-southeast-3" + }, + { + "name": "Asia Pacific (Melbourne)", + "value": "ap-southeast-4" + }, + { + "name": "China (Beijing)", + "value": "cn-north-1" + }, + { + "name": "China (Ningxia)", + "value": "cn-northwest-1" + }, + { + "name": "Africa (Cape Town)", + "value": "af-south-1" + }, + { + "name": "Middle East (Bahrain)", + "value": "me-south-1" + }, + { + "name": "Middle East (UAE)", + "value": "me-central-1" + }, + { + "name": "il-central-1", + "value": "il-central-1" + } + ] }, { "hidden": true, diff --git a/libs/uploaders/backblazeB2.js b/libs/uploaders/backblazeB2.js index c28b0d90..b06294fb 100644 --- a/libs/uploaders/backblazeB2.js +++ b/libs/uploaders/backblazeB2.js @@ -3,18 +3,19 @@ const { Readable } = require('stream'); const B2 = require('backblaze-b2') module.exports = function(s,config,lang){ //Backblaze B2 + var serviceProvider = 'b2' var beforeAccountSaveForBackblazeB2 = function(d){ //d = save event d.formDetails.b2_use_global=d.d.b2_use_global d.formDetails.use_bb_b2=d.d.use_bb_b2 } var cloudDiskUseStartupForBackblazeB2 = function(group,userDetails){ - group.cloudDiskUse['b2'].name = 'Backblaze B2' - group.cloudDiskUse['b2'].sizeLimitCheck = (userDetails.use_bb_b2_size_limit === '1') + group.cloudDiskUse[serviceProvider].name = 'Backblaze B2' + group.cloudDiskUse[serviceProvider].sizeLimitCheck = (userDetails.use_bb_b2_size_limit === '1') if(!userDetails.bb_b2_size_limit || userDetails.bb_b2_size_limit === ''){ - group.cloudDiskUse['b2'].sizeLimit = 10000 + group.cloudDiskUse[serviceProvider].sizeLimit = 10000 }else{ - group.cloudDiskUse['b2'].sizeLimit = parseFloat(userDetails.bb_b2_size_limit) + group.cloudDiskUse[serviceProvider].sizeLimit = parseFloat(userDetails.bb_b2_size_limit) } } var loadBackblazeB2ForUser = function(e){ @@ -83,7 +84,7 @@ module.exports = function(s,config,lang){ }catch(err){ var videoDetails = video.details } - if(video.type !== 'b2'){ + if(video.type !== serviceProvider){ callback() return } @@ -128,30 +129,35 @@ module.exports = function(s,config,lang){ }).then(function(resp){ const uploadResponse = resp.data if(theGroup.init.bb_b2_log === '1' && uploadResponse.fileId){ + const insertDetails = { + bucketId : uploadResponse.bucketId, + fileId : uploadResponse.fileId, + fileName : uploadResponse.fileName + } + const insertQuery = { + mid: e.mid, + ke: e.ke, + time: k.startTime, + status: 1, + type : serviceProvider, + details: insertDetails, + size: k.filesize, + end: k.endTime, + href: '' + } s.knexQuery({ action: "insert", table: "Cloud Videos", - insert: { - mid: e.mid, - ke: e.ke, - time: k.startTime, - status: 1, - type : 'b2', - details: s.s({ - bucketId : uploadResponse.bucketId, - fileId : uploadResponse.fileId, - fileName : uploadResponse.fileName - }), - size: k.filesize, - end: k.endTime, - href: '' - } + insert: Object.assign({},insertQuery,{details: s.s(insertDetails)}) }) s.setCloudDiskUsedForGroup(e.ke,{ amount : k.filesizeMB, - storageType : 'b2' + storageType : serviceProvider + }) + s.purgeCloudDiskForGroup(e,serviceProvider) + s.onCloudVideoUploadedExtensions.forEach((extender) => { + extender(insertQuery) }) - s.purgeCloudDiskForGroup(e,'b2') } }).catch(backblazeErr) }) @@ -183,7 +189,7 @@ module.exports = function(s,config,lang){ } //backblaze b2 s.addCloudUploader({ - name: 'b2', + name: serviceProvider, loadGroupAppExtender: loadBackblazeB2ForUser, unloadGroupAppExtender: unloadBackblazeB2ForUser, insertCompletedVideoExtender: uploadVideoToBackblazeB2, diff --git a/libs/uploaders/s3based.js b/libs/uploaders/s3based.js index 34c053be..7894e08c 100644 --- a/libs/uploaders/s3based.js +++ b/libs/uploaders/s3based.js @@ -324,84 +324,128 @@ module.exports = function(s,config,lang){ "description": "", "default": "", "example": "", - "possible": [ - { - "name": lang['No Region'], - "value": "" - }, - { - "name": "US West 1", - "value": "us-west-1" - }, - { - "name": "US West 2", - "value": "us-west-2" - }, - { - "name": "US East 1", - "value": "us-east-1" - }, - { - "name": "US East 2", - "value": "us-east-2" - }, - { - "name": "Asia Pacific 1", - "value": "ap-south-1" - }, - { - "name": "Asia Pacific 2", - "value": "ap-northeast-2" - }, - { - "name": "Asia Pacific 3", - "value": "ap-northeast-3" - }, - { - "name": "Asia Pacific 4", - "value": "ap-southeast-1" - }, - { - "name": "Asia Pacific 5", - "value": "ap-southeast-2" - }, - { - "name": "Asia Pacific 6", - "value": "ap-northeast-1" - }, - { - "name": "Canada 1", - "value": "ca-central-1" - }, - { - "name": "China 1", - "value": "cn-north-1" - }, - { - "name": "China 1", - "value": "cn-northwest-1" - }, - { - "name": "EU 1", - "value": "eu-central-1" - }, - { - "name": "EU 2", - "value": "eu-west-1" - }, - { - "name": "EU 3", - "value": "eu-west-2" - }, - { - "name": "EU 4", - "value": "eu-west-3" - }, - { - "name": "South America 1", - "value": "sa-east-1" - } - ] + "possible": [ + { + "name": lang['No Region'], + "value": "" + }, + { + "name": "US West (N. California)", + "value": "us-west-1" + }, + { + "name": "US West (Oregon)", + "value": "us-west-2" + }, + { + "name": "US East (Ohio)", + "value": "us-east-2" + }, + { + "name": "US East (N. Virginia)", + "value": "us-east-1" + }, + { + "name": "Canada (Central)", + "value": "ca-central-1" + }, + { + "name": "South America (São Paulo)", + "value": "sa-east-1" + }, + { + "name": "EU (Frankfurt)", + "value": "eu-central-1" + }, + { + "name": "EU (Ireland)", + "value": "eu-west-1" + }, + { + "name": "EU (London)", + "value": "eu-west-2" + }, + { + "name": "EU (Paris)", + "value": "eu-west-3" + }, + { + "name": "Europe (Milan)", + "value": "eu-south-1" + }, + { + "name": "Europe (Spain)", + "value": "eu-south-2" + }, + { + "name": "Europe (Zurich)", + "value": "eu-central-2" + }, + { + "name": "Asia Pacific (Mumbai)", + "value": "ap-south-1" + }, + { + "name": "Asia Pacific (Seoul)", + "value": "ap-northeast-2" + }, + { + "name": "Asia Pacific (Osaka-Local)**", + "value": "ap-northeast-3" + }, + { + "name": "Asia Pacific (Singapore)", + "value": "ap-southeast-1" + }, + { + "name": "Asia Pacific (Sydney)", + "value": "ap-southeast-2" + }, + { + "name": "Asia Pacific (Tokyo)", + "value": "ap-northeast-1" + }, + { + "name": "Asia Pacific (Hong Kong)", + "value": "ap-east-1" + }, + { + "name": "Asia Pacific (Hyderabad)", + "value": "ap-south-2" + }, + { + "name": "Asia Pacific (Jakarta)", + "value": "ap-southeast-3" + }, + { + "name": "Asia Pacific (Melbourne)", + "value": "ap-southeast-4" + }, + { + "name": "China (Beijing)", + "value": "cn-north-1" + }, + { + "name": "China (Ningxia)", + "value": "cn-northwest-1" + }, + { + "name": "Africa (Cape Town)", + "value": "af-south-1" + }, + { + "name": "Middle East (Bahrain)", + "value": "me-south-1" + }, + { + "name": "Middle East (UAE)", + "value": "me-central-1" + }, + { + "name": "il-central-1", + "value": "il-central-1" + } + ] }, { "hidden": true, diff --git a/libs/user.js b/libs/user.js index ea580253..3187c250 100644 --- a/libs/user.js +++ b/libs/user.js @@ -18,6 +18,7 @@ module.exports = function(s,config,lang){ deleteCloudTimelapseFrames, resetAllStorageCounters, } = require("./user/utils.js")(s,config,lang); + require("./user/logger.js")(s,config,lang) let purgeDiskGroup = () => {} const runQuery = async.queue(function(groupKey, callback) { purgeDiskGroup(groupKey,callback) diff --git a/libs/user/logger.js b/libs/user/logger.js new file mode 100644 index 00000000..f26c25a6 --- /dev/null +++ b/libs/user/logger.js @@ -0,0 +1,31 @@ +module.exports = function(s,config,lang){ + s.onVideoAccess((videoRow,user,groupKey,monitorId,ip) => { + s.userLog({ + ke: groupKey, + mid: '$USER', + },{ + type: lang['Video Accessed'], + msg: { + user: { + mail: user.mail, + uid: user.uid, + ip, + }, + video: videoRow + } + }) + }) + s.onLogout((user,groupKey,userId,ip) => { + s.userLog({ + ke: groupKey, + mid: '$USER', + },{ + type: lang['User Logged Out'], + msg: { + mail: user.mail, + uid: user.uid, + ip, + } + }) + }) +} diff --git a/libs/webServerPaths.js b/libs/webServerPaths.js index 104fa3b3..25db79b2 100644 --- a/libs/webServerPaths.js +++ b/libs/webServerPaths.js @@ -107,9 +107,18 @@ module.exports = function(s,config,lang,app,io){ * API : Logout */ app.get(config.webPaths.apiPrefix+':auth/logout/:ke/:id', function (req,res){ - if(s.group[req.params.ke] && s.group[req.params.ke].users[req.params.auth] && s.group[req.params.ke].users[req.params.auth].details){ - delete(s.api[req.params.auth]); - delete(s.group[req.params.ke].users[req.params.auth]); + const groupKey = req.params.ke + const userId = req.params.id + const authToken = req.params.auth + const user = s.group[groupKey] && s.group[groupKey].users[authToken] && s.group[groupKey].users[authToken].details ? s.group[groupKey].users[authToken] : null; + if(user){ + const clientIp = s.getClientIp(req) + const user = s.group[groupKey].users[authToken] + s.onLogoutExtensions.forEach((extender) => { + extender(user,groupKey,userId,clientIp) + }); + delete(s.api[authToken]); + delete(s.group[groupKey].users[authToken]); s.knexQuery({ action: "update", table: "Users", @@ -117,9 +126,9 @@ module.exports = function(s,config,lang,app,io){ auth: '', }, where: [ - ['auth','=',req.params.auth], - ['ke','=',req.params.ke], - ['uid','=',req.params.id], + ['auth','=',authToken], + ['ke','=',groupKey], + ['uid','=',userId], ] }) res.end(s.prettyPrint({ok:true,msg:'You have been logged out, session key is now inactive.'})) @@ -1454,6 +1463,10 @@ module.exports = function(s,config,lang,app,io){ }else{ s.streamMp4FileOverHttp(filePath,req,res,!!req.query.pureStream) } + const clientIp = s.getClientIp(req) + s.onVideoAccessExtensions.forEach((extender) => { + extender(videoRow,user,groupKey,monitorId,clientIp) + }) }else{ s.closeJsonResponse(res,{ ok: false, diff --git a/libs/webServerStreamPaths.js b/libs/webServerStreamPaths.js index cd2a85f5..1d7b8478 100644 --- a/libs/webServerStreamPaths.js +++ b/libs/webServerStreamPaths.js @@ -159,7 +159,7 @@ module.exports = function(s,config,lang,app){ return; } s.checkChildProxy(req.params,function(){ - if(s.group[req.params.ke]&&s.group[req.params.ke].activeMonitors[req.params.id]){ + if(s.group[req.params.ke]&&s.group[req.params.ke].activeMonitors&&s.group[req.params.ke].activeMonitors[req.params.id]){ if(user.permissions.watch_stream==="0"||user.details.sub&&user.details.allmonitors!=='1'&&user.details.monitors.indexOf(req.params.id)===-1){ res.end(user.lang['Not Permitted']) return diff --git a/web/assets/css/dashboard.css b/web/assets/css/dashboard.css index e007b649..7a61b7fd 100644 --- a/web/assets/css/dashboard.css +++ b/web/assets/css/dashboard.css @@ -431,3 +431,8 @@ ul.squeeze { flex-direction: column; height: 100vh; } + +.log-item .msg-tree > ul { + list-style: none; + padding-left: 0; +} diff --git a/web/assets/js/bs5.dashboard-base.js b/web/assets/js/bs5.dashboard-base.js index 0edb4b92..a8d7cc1c 100644 --- a/web/assets/js/bs5.dashboard-base.js +++ b/web/assets/js/bs5.dashboard-base.js @@ -327,11 +327,13 @@ function compileConnectUrl(options){ function jsonToHtmlBlock(target){ var html = '' if(target instanceof Object){ + var html = '
    ' $.each(target,function(key,value){ - html += `
    - ${key} : ${jsonToHtmlBlock(value)} -
    ` + html += ` +
  • ${key} : ${jsonToHtmlBlock(value)}
    • + ` }) + html += '
' }else{ html += `${target}` } @@ -836,7 +838,7 @@ function buildLogRow(v){ ${humanMonitorName}${v.info && v.info.type ? v.info.type : v.mid}
-
${jsonToHtmlBlock(v.info.msg)}
+
${jsonToHtmlBlock(v.info.msg)}
${formattedTime(v.time)} diff --git a/web/assets/js/bs5.startup.js b/web/assets/js/bs5.startup.js index 34f4293d..f6025d52 100644 --- a/web/assets/js/bs5.startup.js +++ b/web/assets/js/bs5.startup.js @@ -172,7 +172,7 @@ onWebSocketEvent(function (d){ cpuIndicatorBar.css('width',cpuPercent) cpuIndicatorPercentText.html(cpuPercent) //ram - var ramPercent = d.ram.percent.toFixed(1) + '%' + var ramPercent = parseFloat(d.ram.percent).toFixed(1) + '%' ramIndicatorBar.css('width',ramPercent) ramIndicatorPercentText.html(ramPercent) ramIndicatorUsed.html(d.ram.used.toFixed(2)) diff --git a/web/assets/js/bs5.timelapseViewer.js b/web/assets/js/bs5.timelapseViewer.js index a2271054..cc6b00c8 100644 --- a/web/assets/js/bs5.timelapseViewer.js +++ b/web/assets/js/bs5.timelapseViewer.js @@ -311,6 +311,13 @@ $(document).ready(function(e){ function downloadTimelapseFrame(frame){ downloadFile(frame.href,frame.filename) } + function buildFileBinUrl(data){ + return apiBaseUrl + '/fileBin/' + data.ke + '/' + data.mid + '/' + data.name + } + function downloadTimelapseVideo(data){ + var downloadUrl = buildFileBinUrl(data) + downloadFile(downloadUrl,data.name) + } function onTimelapseVideoBuildComplete(data){ var saveBuiltVideo = dashboardOptions().switches.timelapseSaveBuiltVideo if(saveBuiltVideo === 1){ diff --git a/web/assets/js/bs5.timeline.js b/web/assets/js/bs5.timeline.js index d4b2cf5f..6b342ba4 100644 --- a/web/assets/js/bs5.timeline.js +++ b/web/assets/js/bs5.timeline.js @@ -189,17 +189,28 @@ $(document).ready(function(){ } } function formatVideosForTimeline(videos){ - var formattedVideos = (videos || []).map((item) => { - var blockColor = timeStripItemColors[item.mid]; + var formattedVideos = []; + (videos || []).forEach((video) => { + var blockColor = timeStripItemColors[video.mid]; ++timeStripItemIncrement; - return { + formattedVideos.push({ id: timeStripItemIncrement, content: ``, style: `background-color: ${blockColor};border-color: ${blockColor}`, - start: item.time, - end: item.end, + start: video.time, + end: video.end, group: 1 - } + }) + video.events.forEach((theEvent) => { + ++timeStripItemIncrement; + formattedVideos.push({ + id: timeStripItemIncrement, + content: ``, + style: `background-color: yellow;border-color: yellow`, + start: theEvent.time, + group: 1 + }) + }) }); return formattedVideos } diff --git a/web/assets/js/bs5.videosTable.js b/web/assets/js/bs5.videosTable.js index d674743a..c803b936 100644 --- a/web/assets/js/bs5.videosTable.js +++ b/web/assets/js/bs5.videosTable.js @@ -94,6 +94,7 @@ $(document).ready(function(e){ }, pagination: true, search: true, + pageList: [10, 25, 50, 100, 1000, 2000], columns: [ { field: 'mid', From 2f72d4e1e83bb601bf8d6bdbcea7f0dc005508b0 Mon Sep 17 00:00:00 2001 From: Moe Date: Thu, 19 Oct 2023 12:27:41 -0700 Subject: [PATCH 038/112] Create installPluginAsDaemon.sh --- tools/installPluginAsDaemon.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tools/installPluginAsDaemon.sh diff --git a/tools/installPluginAsDaemon.sh b/tools/installPluginAsDaemon.sh new file mode 100644 index 00000000..140f5cfd --- /dev/null +++ b/tools/installPluginAsDaemon.sh @@ -0,0 +1,24 @@ +#!/bin/bash +DIR=$(dirname $0) +PLUGIN_DIR="$DIR/../plugins/$1" +SPECIFIED_KEY="$3" +if [ -d "$PLUGIN_DIR" ]; then + INSTALLER_SCRIPT="$PLUGIN_DIR/INSTALL.sh" + if [ "$SPECIFIED_KEY" == "" ]; then + echo "Generating Random Plugin Key" + SPECIFIED_KEY=$(head -c 64 < /dev/urandom | sha256sum | awk '{print substr($1,1,60)}') + else + echo "Using specified Plugin Key" + fi + sudo node $DIR/modifyConfigurationForPlugin.js $1 key= + if [ "$2" == "true" ]; then + cd $INSTALLER_SCRIPT + if [ -f "$INSTALLER_SCRIPT" ]; then + sudo sh $INSTALLER_SCRIPT + else + sudo npm install + fi + fi +else + echo "Plugin not found : $PLUGIN_DIR" +fi From d038de233ff906d6565c618b59fd417a3693822d Mon Sep 17 00:00:00 2001 From: Moe Date: Fri, 20 Oct 2023 17:40:12 -0700 Subject: [PATCH 039/112] set default seconds Inward on raw snap to 5 --- definitions/base.js | 2 +- libs/monitor.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/definitions/base.js b/definitions/base.js index 136b50b1..ce01ec6f 100644 --- a/definitions/base.js +++ b/definitions/base.js @@ -2911,7 +2911,7 @@ module.exports = function(s,config,lang){ "name": "detail=snap_seconds_inward", "field": lang['Delay for Snapshot'], "description": lang[lang["fieldTextSnapSecondsInward"]], - "default": "0" + "default": "5" }, { hidden: true, diff --git a/libs/monitor.js b/libs/monitor.js index 95506de6..6aae00c4 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -158,7 +158,7 @@ module.exports = function(s,config,lang){ var outputOptions = [] var streamDir = s.dir.streams + monitor.ke + '/' + monitor.mid + '/' var url = options.url - var secondsInward = options.secondsInward || '0' + var secondsInward = options.secondsInward || '5' if(secondsInward.length === 1 && !isNaN(secondsInward))secondsInward = '0' + secondsInward; var dynamicTimeout = (secondsInward * 1000) + 5000; if(options.flags)outputOptions.push(options.flags) From b073eab0248849c796a2ba6cbedf08eb31885105 Mon Sep 17 00:00:00 2001 From: Moe Date: Mon, 23 Oct 2023 07:44:13 -0700 Subject: [PATCH 040/112] try fixing docker --- Docker/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/init.sh b/Docker/init.sh index 59347938..99689526 100644 --- a/Docker/init.sh +++ b/Docker/init.sh @@ -58,7 +58,7 @@ if [ "$DB_DISABLE_INCLUDED" = "false" ]; then fi echo "Create database user if it does not exists ..." - mysql -e "source /home/Shinobi/sql/user.sql" || true + mysql -u root -e "source /home/Shinobi/sql/user.sql" || true else echo "Create database schema if it does not exists ..." From 57a17db48712de62e94e287bc0b58e3cfd0982e1 Mon Sep 17 00:00:00 2001 From: Moe Date: Mon, 23 Oct 2023 14:17:22 -0700 Subject: [PATCH 041/112] Performance Updates to Timeline --- definitions/base.js | 3 +- web/assets/css/dashboard.css | 2 +- web/assets/js/bs5.dashboard-base.js | 7 +++- web/assets/js/bs5.timeline.js | 59 +++++++++++++++++++++++++---- web/pages/blocks/home/menuSide.ejs | 2 +- 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/definitions/base.js b/definitions/base.js index ce01ec6f..c6d5007c 100644 --- a/definitions/base.js +++ b/definitions/base.js @@ -7773,6 +7773,7 @@ module.exports = function(s,config,lang){ label: `${lang['Timeline']}`, pageOpen: 'timeline', addUl: true, + ulStyle: `max-height:250px;overflow:auto;` }, { icon: 'map-marker', @@ -9178,7 +9179,7 @@ module.exports = function(s,config,lang){
- +
diff --git a/web/assets/css/dashboard.css b/web/assets/css/dashboard.css index 7a61b7fd..ada936fa 100644 --- a/web/assets/css/dashboard.css +++ b/web/assets/css/dashboard.css @@ -141,6 +141,7 @@ body { box-shadow: 0 0 5px #1e2b37; } +.dot-grey {background:#777} .dot-red {background:#d9534f} .dot-purple {background:#3f51b5} .dot-blue {background:#375182} @@ -148,7 +149,6 @@ body { .dot-green {background:#449d44} .dot-forestgreen {background:#408693} .dot-orange {background:#c49a68} -.dot-grey {background:#777} .slidemenu .nav-link:hover { color: #9abadd!important; diff --git a/web/assets/js/bs5.dashboard-base.js b/web/assets/js/bs5.dashboard-base.js index a8d7cc1c..0243f8fb 100644 --- a/web/assets/js/bs5.dashboard-base.js +++ b/web/assets/js/bs5.dashboard-base.js @@ -728,7 +728,12 @@ function getAllSectionsFromDefinition(definitionsBase){ function buildSubMenuItems(listOfItems){ var html = '' $.each(listOfItems,function(n,item){ - if(item)html += `
  • ${item.label}
    • ` + if(!item)return; + if(item.divider){ + html += `
    • ` + }else{ + html += `
  • ${item.label}
    • ` + } }) return html } diff --git a/web/assets/js/bs5.timeline.js b/web/assets/js/bs5.timeline.js index 6b342ba4..1a10a004 100644 --- a/web/assets/js/bs5.timeline.js +++ b/web/assets/js/bs5.timeline.js @@ -30,7 +30,7 @@ $(document).ready(function(){ var timeStripItemColors = {} var timeStripAutoGridSizer = false var timeStripListOfQueries = [] - var timeStripSelectedMonitors = [] + var timeStripSelectedMonitors = dashboardOptions().timeStripSelectedMonitors || [] var timeStripAutoScrollTimeout = null; var timeStripAutoScrollPositionStart = null; var timeStripAutoScrollPositionEnd = null; @@ -42,12 +42,14 @@ $(document).ready(function(){ var loadedVideoElsOnCanvasNextVideoTimeout = {} var loadedVideoEndingTimeouts = {} var playUntilVideoEnd = false - var dontShowDetectionOnTimeline = false + var dontShowDetectionOnTimeline = true var isPlaying = false var earliestStart = null var latestEnd = null var timeChanging = false var dateRangeChanging = false + var lastDateChecked = new Date(0) + var monitorSelectionElements = [] function setLoadingMask(turnOn){ if(turnOn){ if(theWindow.find('.loading-mask').length === 0){ @@ -127,16 +129,17 @@ $(document).ready(function(){ return videos; } async function getVideosByTimeStripRange(addOrOverWrite){ - // timeStripSelectedMonitors = selected monitors var currentVideosLength = parseInt(loadedVideosOnTimeStrip.length) var stripDate = getTimestripDate() var startDate = stripDate.start var endDate = stripDate.end + var dateNow = new Date() + var isOverCacheTime = dateNow.getTime() - lastDateChecked.getTime() >= 20 * 60 * 1000; + if(isOverCacheTime){ + timeStripListOfQueries = [] + loadedVideosOnTimeStrip = [] + } var gaps = findGapsInSearchRanges(timeStripListOfQueries, [startDate,endDate]) - // console.error([startDate,endDate]) - // console.log('range : ',JSON.stringify([startDate,endDate])) - // console.log('timeRanges : ',JSON.stringify(timeStripListOfQueries)) - // console.log('gaps : ',JSON.stringify(gaps)) if(gaps.length > 0){ setLoadingMask(true) timeStripListOfQueries.push(...gaps) @@ -146,6 +149,7 @@ $(document).ready(function(){ if(currentVideosLength !== loadedVideosOnTimeStrip.length)addTimelineItems(loadedVideosOnTimeStrip); setLoadingMask(false) } + lastDateChecked = new Date(); return loadedVideosOnTimeStrip } function selectVideosForCanvas(time, videos){ @@ -240,6 +244,7 @@ $(document).ready(function(){ setTimeLabel(clickTime) setTimeOfCanvasVideos(clickTime) setHollowClickQueue() + setSideMenuMonitorVisualSelection() } function timeStripActionWithPausePlay(restartPlaySpeed){ return new Promise((resolve,reject) => { @@ -769,6 +774,7 @@ $(document).ready(function(){ } function drawFoundCamerasSubMenu(){ var tags = getListOfTagsFromMonitors() + var monitorsOrdered = Object.values(loadedMonitors).sort((a, b) => a.name.localeCompare(b.name)); var allFound = [ { attributes: `timeline-menu-action="selectMonitorGroup" tag=""`, @@ -792,9 +798,33 @@ $(document).ready(function(){ color: ' d-none', label: `${lang.addTagText}`, }) + }else if(allFound.length !== 0){ + allFound.push({ + divider: true + }) } + $.each(monitorsOrdered,function(monitorKey,monitor){ + var monitorId = monitor.mid + var label = monitor.name + allFound.push({ + attributes: `timeline-menu-action="selectMonitor" data-mid="${monitorId}"`, + class: `cursor-pointer timeline-selectMonitor`, + color: 'grey', + label, + }) + }) var html = buildSubMenuItems(allFound) sideMenuList.html(html) + monitorSelectionElements = sideMenuList.find('.timeline-selectMonitor') + } + function setSideMenuMonitorVisualSelection(){ + var getForAllMonitors = timeStripSelectedMonitors.length === 0; + monitorSelectionElements.find('.dot')[getForAllMonitors ? 'addClass' : 'removeClass']('dot-green') + if(!getForAllMonitors){ + timeStripSelectedMonitors.forEach((monitorId) => { + sideMenuList.find(`[data-mid="${monitorId}"] .dot`).addClass('dot-green') + }) + } } function setColorReferences(){ $.each(loadedMonitors,function(monitorId,monitor){ @@ -853,10 +883,24 @@ $(document).ready(function(){ if(!video)return console.log('No More!') await jumpToVideo(video) } + function onSelectedMonitorChange(){ + dashboardOptions('timeStripSelectedMonitors', timeStripSelectedMonitors) + } sideMenuList.on('click','[timeline-menu-action]',function(){ var el = $(this) var type = el.attr('timeline-menu-action') switch(type){ + case'selectMonitor': + var monitorId = el.attr('data-mid') + var alreadySelected = timeStripSelectedMonitors.indexOf(monitorId) > -1; + if(alreadySelected){ + timeStripSelectedMonitors = timeStripSelectedMonitors.filter(fillId => monitorId !== fillId) + }else{ + timeStripSelectedMonitors.push(monitorId) + } + onSelectedMonitorChange() + refreshTimeline() + break; case'selectMonitorGroup': var tag = el.attr('tag') if(!tag){ @@ -866,6 +910,7 @@ $(document).ready(function(){ var monitorIds = tags[tag] timeStripSelectedMonitors = [...monitorIds]; } + onSelectedMonitorChange() refreshTimeline() break; } diff --git a/web/pages/blocks/home/menuSide.ejs b/web/pages/blocks/home/menuSide.ejs index 4e822626..01de26ec 100644 --- a/web/pages/blocks/home/menuSide.ejs +++ b/web/pages/blocks/home/menuSide.ejs @@ -57,7 +57,7 @@   <%- item.label %> <% if(item.addUl){ %> -