From 459dc7339463227e13f4ece5e4a2ab1726d48126 Mon Sep 17 00:00:00 2001 From: Jarrod Connolly Date: Fri, 15 Oct 2021 09:58:17 +0800 Subject: [PATCH 01/15] ensure camera stream dir exists on master --- libs/monitor.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/monitor.js b/libs/monitor.js index c8a476cd..cf252c98 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -186,6 +186,11 @@ module.exports = function(s,config,lang){ var temporaryImageFile = streamDir + s.gid(5) + '.jpg' var iconImageFile = streamDir + 'icon.jpg' var ffmpegCmd = splitForFFPMEG(`-loglevel warning -re -probesize 100000 -analyzeduration 100000 ${inputOptions.join(' ')} -i "${url}" ${outputOptions.join(' ')} -f image2 -an -vf "fps=1" -vframes 1 "${temporaryImageFile}"`) + checkExists(streamDir, function(success) { + if (success === false) { + fs.mkdirSync(streamDir, {recursive: true}, (err) => {s.debugLog(err)}) + } + }) const snapProcess = new Worker(__dirname + '/cameraThread/snapshot.js', { workerData: { jsonData: { From 35e339761f8cd3c36bef1f1b4475b40dbc607df3 Mon Sep 17 00:00:00 2001 From: Jarrod Connolly Date: Fri, 15 Oct 2021 10:05:38 +0800 Subject: [PATCH 02/15] fix master failing to write as dir not existing --- libs/childNode.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/childNode.js b/libs/childNode.js index e7207d65..6536bff7 100644 --- a/libs/childNode.js +++ b/libs/childNode.js @@ -118,6 +118,9 @@ module.exports = function(s,config,lang,app,io){ case'created_file_chunk': if(!s.group[d.ke].activeMonitors[d.mid].childNodeStreamWriters[d.filename]){ d.dir = s.getVideoDirectory(s.group[d.ke].rawMonitorConfigurations[d.mid]) + if (!fs.existsSync(d.dir)) { + fs.mkdirSync(d.dir, {recursive: true}, (err) => {s.debugLog(err)}) + } s.group[d.ke].activeMonitors[d.mid].childNodeStreamWriters[d.filename] = fs.createWriteStream(d.dir+d.filename) } s.group[d.ke].activeMonitors[d.mid].childNodeStreamWriters[d.filename].write(d.chunk) From 0ac6d371e76b05d0f7ee55cb378048f344eba9ad Mon Sep 17 00:00:00 2001 From: Jarrod Connolly Date: Fri, 15 Oct 2021 10:10:29 +0800 Subject: [PATCH 03/15] delete messages were emitting to wrong childnode --- libs/childNode.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/childNode.js b/libs/childNode.js index 6536bff7..eaf3ba4a 100644 --- a/libs/childNode.js +++ b/libs/childNode.js @@ -38,7 +38,7 @@ module.exports = function(s,config,lang,app,io){ ipAddress = cn.request.connection.remoteAddress.replace('::ffff:','')+':'+d.port cn.ip = ipAddress cn.shinobi_child = 1 - tx = function(z){ + cn.tx = function(z){ cn.emit('c',z) } if(!s.childNodes[cn.ip]){ @@ -52,7 +52,7 @@ module.exports = function(s,config,lang,app,io){ d.availableHWAccels.forEach(function(accel){ if(config.availableHWAccels.indexOf(accel) === -1)config.availableHWAccels.push(accel) }) - tx({ + cn.tx({ f : 'init_success', childNodes : s.childNodes }) @@ -103,7 +103,7 @@ module.exports = function(s,config,lang,app,io){ return console.log('FILE NOT EXIST') } s.group[d.ke].activeMonitors[d.mid].childNodeStreamWriters[d.filename].end() - tx({ + cn.tx({ f: 'deleteTimelapseFrame', file: d.filename, currentDate: d.currentDate, @@ -130,7 +130,7 @@ module.exports = function(s,config,lang,app,io){ return console.log('FILE NOT EXIST') } s.group[d.ke].activeMonitors[d.mid].childNodeStreamWriters[d.filename].end(); - tx({ + cn.tx({ f:'delete', file:d.filename, ke:d.ke, From dc3f94e37ca1cbddca9cbc8867b6a89cc576a783 Mon Sep 17 00:00:00 2001 From: Moe Date: Fri, 15 Oct 2021 13:43:00 -0700 Subject: [PATCH 04/15] allow swapping pam-diff engine --- libs/cameraThread/libs/detectorUtils.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/cameraThread/libs/detectorUtils.js b/libs/cameraThread/libs/detectorUtils.js index 5bcae2e4..ed7f2764 100644 --- a/libs/cameraThread/libs/detectorUtils.js +++ b/libs/cameraThread/libs/detectorUtils.js @@ -1,6 +1,7 @@ const P2P = require('pipe2pam') -const PamDiff = require('pam-diff') -module.exports = function(jsonData,pamDiffResponder){ +let PamDiff = require('pam-diff') +module.exports = function(jsonData,pamDiffResponder,alternatePamDiff){ + if(alternatePamDiff)PamDiff = alternatePamDiff; const noiseFilterArray = {}; const config = jsonData.globalInfo.config const completeMonitorConfig = jsonData.rawMonitorConfig @@ -390,5 +391,6 @@ module.exports = function(jsonData,pamDiffResponder){ regionConfidenceMaximums, regionTriggerThresholds, mergeTriggers, + pamDiff, } } From e89ad75f71023aa5bfd7db20f81c67330bb24c3a Mon Sep 17 00:00:00 2001 From: Moe Date: Sat, 16 Oct 2021 15:46:13 -0700 Subject: [PATCH 05/15] add stimeout to snapshot ffmpeg command --- libs/monitor/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/monitor/utils.js b/libs/monitor/utils.js index ea2b5698..cfcf8446 100644 --- a/libs/monitor/utils.js +++ b/libs/monitor/utils.js @@ -136,7 +136,7 @@ module.exports = (s,config,lang) => { }) } const temporaryImageFile = streamDir + s.gid(5) + '.jpg' - const ffmpegCmd = splitForFFPMEG(`-loglevel warning -re -probesize 100000 -analyzeduration 100000 ${inputOptions.join(' ')} -i "${url}" ${outputOptions.join(' ')} -f image2 -an -vf "fps=1" -vframes 1 "${temporaryImageFile}"`) + const ffmpegCmd = splitForFFPMEG(`-loglevel warning -re -stimeout 30000000 -probesize 100000 -analyzeduration 100000 ${inputOptions.join(' ')} -i "${url}" ${outputOptions.join(' ')} -f image2 -an -vf "fps=1" -vframes 1 "${temporaryImageFile}"`) const snapProcess = spawn('ffmpeg',ffmpegCmd,{detached: true}) snapProcess.stderr.on('data',function(data){ // s.debugLog(data.toString()) From df74cef008ac0d4a6fab1fbc33146138b12f4dfc Mon Sep 17 00:00:00 2001 From: Moe Date: Sun, 17 Oct 2021 22:04:21 -0700 Subject: [PATCH 06/15] get snapshot from ONVIF if initialized + minor cleaning --- libs/control/onvif.js | 15 +++++++++++++++ libs/monitor.js | 27 +++++++++++++++++++++------ libs/scanners/utils.js | 15 ++++----------- libs/webServerAdminPaths.js | 8 +++++++- 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/libs/control/onvif.js b/libs/control/onvif.js index 745a1f3e..77703fe4 100644 --- a/libs/control/onvif.js +++ b/libs/control/onvif.js @@ -2,6 +2,10 @@ var os = require('os'); var exec = require('child_process').exec; const onvif = require("shinobi-onvif"); module.exports = function(s,config,lang,app,io){ + const { + createSnapshot, + addCredentialsToStreamLink, + } = require('../monitor/utils.js')(s,config,lang) const createOnvifDevice = async (onvifAuth) => { var response = {ok: false} const monitorConfig = s.group[onvifAuth.ke].rawMonitorConfigurations[onvifAuth.id] @@ -118,6 +122,16 @@ module.exports = function(s,config,lang,app,io){ doAction(s.group[onvifAuth.ke].activeMonitors[onvifAuth.id].onvifConnection) } } + async function getSnapshotFromOnvif(onvifOptions){ + return await createSnapshot({ + output: ['-s 400x400'], + url: addCredentialsToStreamLink({ + username: onvifOptions.username, + password: onvifOptions.password, + url: onvifOptions.uri + }), + }) + } /** * API : ONVIF Method Controller */ @@ -140,6 +154,7 @@ module.exports = function(s,config,lang,app,io){ }) },res,req); }) + s.getSnapshotFromOnvif = getSnapshotFromOnvif s.createOnvifDevice = createOnvifDevice s.runOnvifMethod = runOnvifMethod } diff --git a/libs/monitor.js b/libs/monitor.js index cf252c98..dc469b6e 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -500,9 +500,12 @@ module.exports = function(s,config,lang){ s.checkDetails(e) if(e.ke && config.doSnapshot === true){ if(s.group[e.ke] && s.group[e.ke].rawMonitorConfigurations && s.group[e.ke].rawMonitorConfigurations[e.mid] && s.group[e.ke].rawMonitorConfigurations[e.mid].mode !== 'stop'){ - var pathDir = s.dir.streams+e.ke+'/'+e.mid+'/' - const {screenShot, isStaticFile} = await s.getRawSnapshotFromMonitor(s.group[e.ke].rawMonitorConfigurations[e.mid],options) - if(screenShot){ + if(s.group[e.ke].activeMonitors[e.mid].onvifConnection){ + const screenShot = await s.getSnapshotFromOnvif({ + username: onvifUsername, + password: onvifPassword, + uri: cameraResponse.uri, + }); s.tx({ f: 'monitor_snapshot', snapshot: screenShot.toString('base64'), @@ -511,9 +514,21 @@ module.exports = function(s,config,lang){ ke: e.ke },'GRP_'+e.ke) }else{ - s.debugLog('Damaged Snapshot Data') - s.tx({f:'monitor_snapshot',snapshot:e.mon.name,snapshot_format:'plc',mid:e.mid,ke:e.ke},'GRP_'+e.ke) - } + var pathDir = s.dir.streams+e.ke+'/'+e.mid+'/' + const {screenShot, isStaticFile} = await s.getRawSnapshotFromMonitor(s.group[e.ke].rawMonitorConfigurations[e.mid],options) + if(screenShot){ + s.tx({ + f: 'monitor_snapshot', + snapshot: screenShot.toString('base64'), + snapshot_format: 'b64', + mid: e.mid, + ke: e.ke + },'GRP_'+e.ke) + }else{ + s.debugLog('Damaged Snapshot Data') + s.tx({f:'monitor_snapshot',snapshot:e.mon.name,snapshot_format:'plc',mid:e.mid,ke:e.ke},'GRP_'+e.ke) + } + } }else{ s.tx({f:'monitor_snapshot',snapshot:'Disabled',snapshot_format:'plc',mid:e.mid,ke:e.ke},'GRP_'+e.ke) } diff --git a/libs/scanners/utils.js b/libs/scanners/utils.js index 764ffb95..4cd4f8f2 100644 --- a/libs/scanners/utils.js +++ b/libs/scanners/utils.js @@ -4,10 +4,6 @@ const { stringContains, } = require('../common.js') module.exports = (s,config,lang) => { - const { - createSnapshot, - addCredentialsToStreamLink, - } = require('../monitor/utils.js')(s,config,lang) const ipRange = (start_ip, end_ip) => { var startLong = toLong(start_ip); var endLong = toLong(end_ip); @@ -148,13 +144,10 @@ module.exports = (s,config,lang) => { var imageSnap if(cameraResponse.uri){ try{ - imageSnap = (await createSnapshot({ - output: ['-s 400x400'], - url: addCredentialsToStreamLink({ - username: onvifUsername, - password: onvifPassword, - url: cameraResponse.uri - }), + imageSnap = (await s.getSnapshotFromOnvif({ + username: onvifUsername, + password: onvifPassword, + uri: cameraResponse.uri, })).toString('base64'); }catch(err){ s.debugLog(err) diff --git a/libs/webServerAdminPaths.js b/libs/webServerAdminPaths.js index 18ebfb1f..55f78259 100644 --- a/libs/webServerAdminPaths.js +++ b/libs/webServerAdminPaths.js @@ -302,7 +302,13 @@ module.exports = function(s,config,lang,app){ } }else{ if(!user.details.sub || user.details.allmonitors === '1' || user.details.monitor_edit.indexOf(req.params.id) > -1 || hasRestrictions && user.details.monitor_create === '1'){ - s.userLog(s.group[req.params.ke].rawMonitorConfigurations[req.params.id],{type:'Monitor Deleted',msg:'by user : '+user.uid}); + s.userLog({ + ke: req.params.ke, + mid: req.params.id + },{ + type: 'Monitor Deleted', + msg: 'by user : '+user.uid + }); req.params.delete=1;s.camera('stop',req.params); s.tx({f:'monitor_delete',uid:user.uid,mid:req.params.id,ke:req.params.ke},'GRP_'+req.params.ke); s.knexQuery({ From 80132ec4950c6162b0410756ff07201a43bd580f Mon Sep 17 00:00:00 2001 From: Moe Date: Sun, 17 Oct 2021 22:07:48 -0700 Subject: [PATCH 07/15] critical update to onvif scanner : fixes import --- web/libs/js/dash2.onvifscanner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/libs/js/dash2.onvifscanner.js b/web/libs/js/dash2.onvifscanner.js index 0a44a16e..adc85f63 100644 --- a/web/libs/js/dash2.onvifscanner.js +++ b/web/libs/js/dash2.onvifscanner.js @@ -53,7 +53,7 @@ $(document).ready(function(e){ mid: tempID + `${options.port}`, host: pathLocation.hostname, port: pathLocation.port, - path: pathLocation.pathname, + path: pathLocation.pathname + (pathLocation.search && pathLocation.search !== '?' ? pathLocation.search : ''), protocol: theLocation.protocol, details: { auto_host: addCredentialsToUri(streamUrl,currentUsername,currentPassword), From 157830022586bd89cad1dacd7c7ad266c0f2f9d6 Mon Sep 17 00:00:00 2001 From: Ian Rubado Date: Thu, 28 Oct 2021 15:37:15 -0400 Subject: [PATCH 08/15] Logic fix for object detection and motion. Allows for motion first to work with yes and no. Logic fix for object detection and motion. Allows for motion first to work with yes and no. --- libs/monitor.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libs/monitor.js b/libs/monitor.js index dc469b6e..0d57dbb4 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -950,11 +950,15 @@ module.exports = function(s,config,lang){ console.log(err) } }) - if(e.details.detector_use_detect_object === '1'){ + if(e.details.detector_use_detect_object === '1' && e.details.detector_use_motion === '1' ){ s.group[e.ke].activeMonitors[e.id].spawn.stdio[4].on('data',function(data){ onDetectorJpegOutputSecondary(e,data) }) - } + }else{ + s.group[e.ke].activeMonitors[e.id].spawn.stdio[4].on('data',function(data){ + onDetectorJpegOutputAlone(e,data) + }) + } }else if(e.details.detector_use_detect_object === '1' && e.details.detector_send_frames !== '1'){ s.group[e.ke].activeMonitors[e.id].spawn.stdio[4].on('data',function(data){ onDetectorJpegOutputSecondary(e,data) From a316f860835dadb67e47bab672695f53ca766fb6 Mon Sep 17 00:00:00 2001 From: Bar Date: Sat, 30 Oct 2021 16:11:36 +0300 Subject: [PATCH 09/15] Updated images to to NodeJS 16 + NVidia CUDA --- Dockerfile | 4 +- Dockerfile.nvidia | 118 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.nvidia diff --git a/Dockerfile b/Dockerfile index 853f97ed..dc0ede43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12.21.0-buster-slim +FROM node:16.13-buster-slim ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -108,4 +108,4 @@ EXPOSE 8080 443 21 25 ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] -CMD [ "pm2-docker", "pm2.yml" ] +CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia new file mode 100644 index 00000000..0a06510b --- /dev/null +++ b/Dockerfile.nvidia @@ -0,0 +1,118 @@ +FROM nvidia/cuda:11.4.2-cudnn8-runtime-ubuntu20.04 + +ENV DB_USER=majesticflame \ + DB_PASSWORD='' \ + DB_HOST='localhost' \ + DB_DATABASE=ccio \ + DB_PORT=3306 \ + SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ + PLUGIN_KEYS='{}' \ + SSL_ENABLED='false' \ + SSL_COUNTRY='CA' \ + SSL_STATE='BC' \ + SSL_LOCATION='Vancouver' \ + SSL_ORGANIZATION='Shinobi Systems' \ + SSL_ORGANIZATION_UNIT='IT Department' \ + SSL_COMMON_NAME='nvr.ninja' \ + DB_DISABLE_INCLUDED=false +ARG DEBIAN_FRONTEND=noninteractive + +RUN mkdir -p /home/Shinobi /config /var/lib/mysql + +RUN apt update -y +RUN apt install wget curl net-tools -y + +# Additional packages to support NodeJS as the base build is NVidia CUDA +RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - +RUN apt install nodejs -y + +RUN node -v +RUN npm -v + +# Install MariaDB server... the debian way +RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ + { \ + echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ + echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ + } | debconf-set-selections; \ + apt-get update; \ + apt-get install -y \ + "mariadb-server" \ + socat \ + ; \ + find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi + +RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi + +# Install FFmpeg + +RUN apt update --fix-missing +RUN apt install -y software-properties-common \ + libfreetype6-dev \ + libgnutls28-dev \ + libmp3lame-dev \ + libass-dev \ + libogg-dev \ + libtheora-dev \ + libvorbis-dev \ + libvpx-dev \ + libwebp-dev \ + libssh2-1-dev \ + libopus-dev \ + librtmp-dev \ + libx264-dev \ + libx265-dev \ + yasm +RUN apt install -y \ + build-essential \ + bzip2 \ + coreutils \ + procps \ + gnutls-bin \ + nasm \ + tar \ + x264 + +RUN apt install -y \ + ffmpeg \ + git \ + make \ + g++ \ + gcc \ + pkg-config \ + python3 \ + wget \ + tar \ + sudo \ + xz-utils + + +WORKDIR /home/Shinobi +COPY . . +#RUN rm -rf /home/Shinobi/plugins +COPY ./plugins /home/Shinobi/plugins +RUN chmod -R 777 /home/Shinobi/plugins +RUN npm i npm@latest -g && \ + npm install --unsafe-perm && \ + npm install pm2 -g +COPY ./Docker/pm2.yml ./ + +# Copy default configuration files +# COPY ./config/conf.json ./config/super.json /home/Shinobi/ +RUN chmod -f +x /home/Shinobi/Docker/init.sh +RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh +# RUN chmod -f +x /home/Shinobi/shinobi + +VOLUME ["/home/Shinobi/videos"] +VOLUME ["/home/Shinobi/plugins"] +VOLUME ["/config"] +VOLUME ["/customAutoLoad"] +VOLUME ["/var/lib/mysql"] + +EXPOSE 8080 443 21 25 + +ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] + +CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] From 5b9b624ebffe5cf0ca9b10627766e81fcbd3308f Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Sat, 30 Oct 2021 16:25:39 +0000 Subject: [PATCH 10/15] Update .gitlab-ci.yml --- .gitlab-ci.yml | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..599b3e2a --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,93 @@ +# This file is a template, and might need editing before it works on your project. +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml + +# Build a Docker image with CI/CD and push to the GitLab registry. +# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html +# +# This template uses one generic job with conditional builds +# for the default branch and all other (MR) branches. + +docker-latest-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag="" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag=":$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + +docker-arm32v7-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag=":arm32v7" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm32v7'" + else + tag=":arm32v7-$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile.arm32v7 + + +docker-nvidia-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag=":nvidia" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'nvidia'" + else + tag=":nvidia-$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile.nvidia From 1b29dc3f4c3918738bb87e9a62c0a4c5ea93843f Mon Sep 17 00:00:00 2001 From: Moe Date: Thu, 4 Nov 2021 19:14:42 -0700 Subject: [PATCH 11/15] Fix Backblaze's response changes --- libs/uploaders/backblazeB2.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libs/uploaders/backblazeB2.js b/libs/uploaders/backblazeB2.js index 0ca101b5..9bc432d4 100644 --- a/libs/uploaders/backblazeB2.js +++ b/libs/uploaders/backblazeB2.js @@ -53,9 +53,9 @@ module.exports = function(s,config,lang){ applicationKey: userDetails.bb_b2_applicationKey }) b2.authorize().then(function(resp){ - s.group[e.ke].bb_b2_downloadUrl = resp.data.downloadUrl + s.group[e.ke].bb_b2_downloadUrl = resp.downloadUrl b2.listBuckets().then(function(resp){ - var buckets = resp.data.buckets + var buckets = resp.buckets var bucketN = -2 buckets.forEach(function(item,n){ if(item.bucketName === userDetails.bb_b2_bucket){ @@ -69,7 +69,7 @@ module.exports = function(s,config,lang){ userDetails.bb_b2_bucket, 'allPublic' ).then(function(resp){ - s.group[e.ke].bb_b2_bucketId = resp.data.bucketId + s.group[e.ke].bb_b2_bucketId = resp.bucketId }).catch(backblazeErr) } }).catch(backblazeErr) @@ -98,7 +98,7 @@ module.exports = function(s,config,lang){ fileId: videoDetails.fileId, fileName: videoDetails.fileName }).then(function(resp){ - // console.log('deleteFileVersion',resp.data) + // console.log('deleteFileVersion',resp) }).catch(function(err){ console.log('deleteFileVersion',err) }) @@ -117,7 +117,7 @@ module.exports = function(s,config,lang){ var backblazeSavePath = s.group[e.ke].init.bb_b2_dir+e.ke+'/'+e.mid+'/'+k.filename var getUploadUrl = function(bucketId,callback){ s.group[e.ke].bb_b2.getUploadUrl(bucketId).then(function(resp){ - callback(resp.data) + callback(resp) }).catch(backblazeErr) } getUploadUrl(s.group[e.ke].bb_b2_bucketId,function(req){ @@ -128,7 +128,7 @@ module.exports = function(s,config,lang){ data: data, onUploadProgress: null }).then(function(resp){ - if(s.group[e.ke].init.bb_b2_log === '1' && resp.data.fileId){ + if(s.group[e.ke].init.bb_b2_log === '1' && resp.fileId){ var backblazeDownloadUrl = s.group[e.ke].bb_b2_downloadUrl + '/file/' + s.group[e.ke].init.bb_b2_bucket + '/' + backblazeSavePath s.knexQuery({ action: "insert", @@ -140,9 +140,9 @@ module.exports = function(s,config,lang){ status: 1, details: s.s({ type : 'b2', - bucketId : resp.data.bucketId, - fileId : resp.data.fileId, - fileName : resp.data.fileName + bucketId : resp.bucketId, + fileId : resp.fileId, + fileName : resp.fileName }), size: k.filesize, end: k.endTime, From 3eae0b64c768a501c8ce4a51689abb8105854a90 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Fri, 5 Nov 2021 10:32:45 +0000 Subject: [PATCH 12/15] Fix wrong SQL exeuction on startup --- Docker/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/init.sh b/Docker/init.sh index 31a18864..64b6c1db 100644 --- a/Docker/init.sh +++ b/Docker/init.sh @@ -69,7 +69,7 @@ if [ "$DB_DISABLE_INCLUDED" = "false" ]; then else echo "Create database schema if it does not exists ..." - mysql -u "$DB_USER" -h "$DB_HOST" -p"$DB_PASSWORD" --port="$DB_PORT" --database="$DB_DATABASE" -e "source /home/Shinobi/sql/framework.sql" || true + mysql -u "$DB_USER" -h "$DB_HOST" -p"$DB_PASSWORD" --port="$DB_PORT" --database="$DB_DATABASE" -e "source /home/Shinobi/sql/framework1.sql" || true fi DATABASE_CONFIG='{"host": "'$DB_HOST'","user": "'$DB_USER'","password": "'$DB_PASSWORD'","database": "'$DB_DATABASE'","port":'$DB_PORT'}' From 318b5940a23ddb9860278619269c5288c52240e2 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Fri, 5 Nov 2021 19:34:50 +0000 Subject: [PATCH 13/15] Fix VOLUME /customAutoLoad --- Dockerfile | 2 +- Dockerfile.arm32v7 | 2 +- Dockerfile.nvidia | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc0ede43..e0b40bde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,8 +100,8 @@ RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/plugins"] +VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/customAutoLoad"] VOLUME ["/var/lib/mysql"] EXPOSE 8080 443 21 25 diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 index e7bd2698..c85896f6 100644 --- a/Dockerfile.arm32v7 +++ b/Dockerfile.arm32v7 @@ -98,8 +98,8 @@ RUN chmod -f +x /home/Shinobi/Docker/init.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/plugins"] +VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/customAutoLoad"] VOLUME ["/var/lib/mysql"] EXPOSE 8080 diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia index 0a06510b..10ac76ef 100644 --- a/Dockerfile.nvidia +++ b/Dockerfile.nvidia @@ -107,8 +107,8 @@ RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/plugins"] +VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/customAutoLoad"] VOLUME ["/var/lib/mysql"] EXPOSE 8080 443 21 25 From bb010997d1ff86f98a04f4c1bbaefc39304467ca Mon Sep 17 00:00:00 2001 From: Matheus de Paula Date: Sun, 14 Nov 2021 02:48:15 +0000 Subject: [PATCH 14/15] Update pt_BR.json --- languages/pt_BR.json | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/languages/pt_BR.json b/languages/pt_BR.json index 34b64f42..14367af0 100644 --- a/languages/pt_BR.json +++ b/languages/pt_BR.json @@ -51,7 +51,7 @@ "Enable Nightvision": "Ativar Visão Noturna", "Disable Nightvision": "Desativar Visão Noturna", "Current": "Atual", - "Monitors": "Monitors", + "Monitors": "Monitores", "Video": "Video", "Videos": "Videos", "Events": "Eventos", @@ -122,12 +122,12 @@ "Execute Command": "Executar Comando", "for Global Access": "para Acesso Global", "Help": "Ajuda", - "Don't show this anymore": "Não mostre isso mais", + "Don't show this anymore": "Não me mostre isso mais", "Chat on Discord": "Conversar no Discord", "Documentation": "Documentação", "All Monitors": "Todos Monitores", - "Motion Meter": "Medidos de Movimento", - "FFmpegTip": "FFprobe is a simple multimedia streams analyzer. You can use it to output all kinds of information about an input including duration, frame rate, frame size, etc.", + "Motion Meter": "Medição de movimentos", + "FFmpegTip": "FFprobe é um simples analisador de streams. Você pode usá-lo para extrar todos os tipos de informação sobre uma entrada incluindo duração, taxa de frames por segundo, tamanho do frame, etc;", "Complete Stream URL": "URL de transmissão completa", "ONVIF Scanner": "ONVIF Scanner", "Scan Settings": "Configurações de digitalização", @@ -162,7 +162,7 @@ "Browser Console Log": "Navegador de logs", "All Monitors and Privileges": "Todos monitores e privilégios", "Permissions": "Permissões", - "Time-lapse Tool": "Ferramenta Time-laps", + "Time-lapse Tool": "Time-lapse", "total": "total", "MB": "MB", "Calendar": "Calendário", @@ -187,11 +187,11 @@ "Add New": "Adicionar novo", "Delete Selected Videos": "Excluir vídeos selecionados", "DeleteSelectedVideosMsg": "Deseja excluir esses vídeos? Você não poderá recuperá-los.", - "clientStreamFailedattemptingReconnect": "A verificação ctream do lado do cliente falhou, tentando reconectar.", + "clientStreamFailedattemptingReconnect": "A verificação do stream falhou, tentando se reconectar.", "Delete Filter": "Excluir filtro", "confirmDeleteFilter": "Deseja excluir este filtro? Você não poderá recuperá-lo.", "Fix Video": "Corrigir Vídeo", - "FixVideoMsg": "Você deseja corrigir esse vídeo? Você não poderá desfazer essa ação..", + "FixVideoMsg": "Você deseja corrigir o vídeo? Você não poderá desfazer essa ação..", "DeleteVideoMsg": "Deseja excluir este vídeo? Você não poderá recuperá-lo.", "dropBoxSuccess": "Sucesso! Arquivos salvos em seu Dropbox.", "API Key Deleted": "Chave da API excluída", @@ -246,13 +246,13 @@ "Connected": "Conectado", "Not Connected": "Não conectado", "Lisence Plate Detector": "Detector de placas", - "OpenCV Cascades": "OpenCV Cascatas", + "OpenCV Cascades": "Cascatas OpenCV ", "Refresh List of Cascades": "Atualizar Lista de Cascatas", "\"No Motion\" Detector": "\"Sem movimento\" Detector", "Control": "Controle", "Grouping": "Agrupando   Adicione grupos em Configurações", "Logging": "Logging", - "IdentityText1": "É assim que o sistema irá identificar os dados para este fluxo. Você não pode alterar o ID do Monitor uma vez que você pressionou salvar. Se você quiser, você pode fazer o ID do Monitor mais legível para humanos antes de continuar.", + "IdentityText1": "É assim que o sistema irá identificar os dados para este fluxo. Você não pode alterar o ID do Monitor uma vez que você o salvou. Se você quiser, você pode fazer o ID do Monitor mais legível para humanos antes de continuar.", "IdentityText2": "Você pode duplicar um monitor modificando o ID do Monitor e depois pressionando salvar. Você não pode usar o ID de um monitor que já existe ou ele economizará sobre as informações do banco de dados desse monitor.", "noSpecialCharacters": "Sem espaços ou caracteres especiais.", "NotesPlacholder": "Comentários que você quer deixar para as configurações desta câmera.", @@ -273,7 +273,7 @@ "Path": "Caminho", "Monitor Capture Rate": "Taxa de captura do monitor (FPS)", "Analyzation Duration": "Duração da análise", - "Probe Size": "Probe Size", + "Probe Size": "Tamanho da sonda", "Stream Type": "Tipo de transmissão", "# of Allow MJPEG Clients": "# para permitir clientes MJPEG 0 para infinito", "HLS Video Encoder": "Codificador de vídeo HLS", @@ -281,9 +281,9 @@ "HLS Segment Length": "Comprimento do segmento HLS em segundos", "HLS Preset": "Pré-definição HLS", "HLS List Size": "Tamanho da lista HLS", - "Check Signal Interval": "Verifique o intervalo do sinal em minutos", + "Check Signal Interval": "Verificar o intervalo do sinal em minutos", "Log Signal Event": "Evento de sinal de registro Apenas cliente", - "Quality": "Qualidade 1 para alta, 23 para Low", + "Quality": "Qualidade 1 para alta, 23 para baixa", "Rate": "Taxa (FPS)", "Width": "Largura", "Height": "Altura", @@ -444,13 +444,13 @@ "Process Unexpected Exit": "Saída inesperado do processo", "Process Crashed for Monitor": "Processo de Monitor quebrado", "FFmpegCantStart": "FFmpeg não pôde iniciar", - "FFmpegCantStartText": "O mecanismo de gravação para esta câmera não pôde começar. Pode haver algo errado com a configuração da sua câmera. Se houver algum registro diferente deste, por favor, coloque-os em Problemas no Github.", + "FFmpegCantStartText": "O FFMpeg não pode inicializar. Pode haver algo de errado com a configuração da sua câmera. Se houver alguma configuração diferente, por favor, reporte no Gitlab.", "JPEG Error": "Erro JPEG", "JPEGErrorText": "Houve um problema ao obter dados da sua câmera.", "Fatal Maximum Reached": "Máximo atingido, parando câmera.", - "FatalMaximumReachedText": "Erro JPEG fatal.", + "FatalMaximumReachedText": "Erro fatal de JPEG", "Incorrect Settings Chosen": "Configuração incorreta escolhida", - "Can't Connect": "Não pode conectar", + "Can't Connect": "Não foi possível conectar", "Video Finished": "Vídeo finalizado", "No Monitor Found, Ignoring Request": "Monitor não encontrado, ignorando requisição", "Event": "Evento", From 9648f8cc60bfb246103694f5f06e0e38cca1da12 Mon Sep 17 00:00:00 2001 From: Moe Date: Thu, 18 Nov 2021 19:48:13 -0800 Subject: [PATCH 15/15] update P2P Server list for dev branch --- libs/commander.js | 137 ++++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 73 deletions(-) diff --git a/libs/commander.js b/libs/commander.js index 7ed6d364..96f8b519 100644 --- a/libs/commander.js +++ b/libs/commander.js @@ -1,85 +1,76 @@ const { Worker } = require('worker_threads'); module.exports = function(s,config,lang,app){ + const fetch = require('node-fetch') const { modifyConfiguration, getConfiguration } = require('./system/utils.js')(config) + let customerServerList = !!config.p2pServerList; var runningWorker; config.machineId = config.p2pApiKey + '' + config.p2pGroupId config.p2pTargetAuth = config.p2pTargetAuth || s.gid(30) - if(!config.p2pServerList)config.p2pServerList = { - "vancouver-1": { - name: 'Vancouver-1', - host: 'p2p-vancouver-1.shinobi.cloud', - p2pPort: '8084', - webPort: '8000', - maxNetworkSpeed: { - up: 5000, - down: 5000, - shared: true + if(!config.workerStreamOutHandlers){ + config.workerStreamOutHandlers = [ + 'Base64', + 'FLV', + 'MP4', + ]; + } + if(!customerServerList){ + config.p2pServerList = { + "vancouver-1": { + name: 'Vancouver-1', + host: 'p2p-vancouver-1.shinobi.cloud', + p2pPort: '8084', + webPort: '8000', + maxNetworkSpeed: { + up: 5000, + down: 5000, + shared: true + }, + location: { + lat: 49.284966, + lon: -123.1140607 + } }, - location: { - lat: 49.284966, - lon: -123.1140607 - } - }, - "vancouver-2": { - name: 'Vancouver-2', - host: 'p2p-vancouver-2.shinobi.cloud', - p2pPort: '8084', - webPort: '8000', - maxNetworkSpeed: { - up: 400, - down: 1000, - shared: true + "toronto-1": { + name: 'Toronto-1', + host: 'p2p-toronto-1.shinobi.cloud', + p2pPort: '8084', + webPort: '8000', + maxNetworkSpeed: { + up: 5000, + down: 5000, + shared: true + }, + location: { + lat: 43.644773, + lon: -79.3862837 + } }, - location: { - lat: 49.284966, - lon: -123.1140607 + "paris-1": { + name: 'Paris-1', + host: 'p2p-paris-1.shinobi.cloud', + p2pPort: '8084', + webPort: '8000', + maxNetworkSpeed: { + up: 200, + down: 200, + shared: true + }, + location: { + lat: 48.873877, + lon: 2.295533 + } } - }, - "toronto-1": { - name: 'Toronto-1', - host: 'p2p-toronto-1.shinobi.cloud', - p2pPort: '8084', - webPort: '8000', - maxNetworkSpeed: { - up: 5000, - down: 5000, - shared: true - }, - location: { - lat: 43.644773, - lon: -79.3862837 - } - }, - "paris-1": { - name: 'Paris-1', - host: 'p2p-paris-1.shinobi.cloud', - p2pPort: '8084', - webPort: '8000', - maxNetworkSpeed: { - up: 200, - down: 200, - shared: true - }, - location: { - lat: 48.873877, - lon: 2.295533 - } - }, - "amsterdam-1": { - name: 'Amsterdam-1', - host: 'p2p-amsterdam-1.shinobi.cloud', - p2pPort: '8084', - webPort: '8000', - maxNetworkSpeed: { - up: 500, - down: 500, - shared: true - }, - location: { - lat: 52.348773, - lon: 4.8846043 - } - }, + } + // get latest + fetch('https://cdn.shinobi.video/configs/p2pServers.js') + .then(res => res.text()) + .then((text) => { + try{ + eval(`config.p2pServerList = ` + text) + }catch(err){ + s.debugLog(err) + } + }); } if(!config.p2pHostSelected)config.p2pHostSelected = 'paris-1' const stopWorker = () => {