From 4d71c6836d75f2f100354688700fac82c6b9a67c Mon Sep 17 00:00:00 2001 From: Moe Date: Thu, 19 Oct 2023 15:32:38 +0000 Subject: [PATCH 1/6] Additional Density - Performance Upgrades and Bug Fixes --- .gitlab-ci.yml | 246 +++++++++++++++++++++------ Docker/install_ffmpeg.sh | 21 +++ Docker/install_mariadb.sh | 19 +++ Docker/install_nodejs.sh | 21 +++ Dockerfile | 108 ++++-------- Dockerfile.arm32v7 | 109 ------------ Dockerfile.nodb | 97 ----------- Dockerfile.nvidia | 118 ------------- definitions/base.js | 3 +- languages/en_CA.json | 2 + libs/dataPort.js | 2 + libs/events/utils.js | 7 +- libs/extenders.js | 3 + libs/ffmpeg/builders.js | 5 +- libs/ffmpeg/possibleWarnings.js | 7 +- libs/ffmpeg/utils.js | 13 ++ libs/monitor.js | 13 +- libs/monitor/utils.js | 30 ++-- libs/notifications/webhook.js | 24 ++- libs/uploaders/amazonS3.js | 190 +++++++++++++-------- libs/uploaders/backblazeB2.js | 52 +++--- libs/uploaders/s3based.js | 200 +++++++++++++--------- libs/user.js | 1 + libs/user/logger.js | 31 ++++ libs/webServerPaths.js | 25 ++- libs/webServerStreamPaths.js | 2 +- web/assets/css/dashboard.css | 5 + web/assets/js/bs5.dashboard-base.js | 10 +- web/assets/js/bs5.startup.js | 2 +- web/assets/js/bs5.timelapseViewer.js | 7 + web/assets/js/bs5.timeline.js | 23 ++- web/assets/js/bs5.videosTable.js | 1 + 32 files changed, 731 insertions(+), 666 deletions(-) create mode 100644 Docker/install_ffmpeg.sh create mode 100644 Docker/install_mariadb.sh create mode 100644 Docker/install_nodejs.sh delete mode 100644 Dockerfile.arm32v7 delete mode 100644 Dockerfile.nodb delete mode 100644 Dockerfile.nvidia create mode 100644 libs/user/logger.js diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79d095a1..97d8f284 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,37 +1,56 @@ -# This file is a template, and might need editing before it works on your project. -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml - -# Build a Docker image with CI/CD and push to the GitLab registry. -# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html -# -# This template uses one generic job with conditional builds -# for the default branch and all other (MR) branches. - docker-latest-build: - # Use the official docker image. image: docker:latest stage: build + variables: + EXCLUDE_DB: "false" + BASE_IMAGE: "node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag="" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" - else - tag=":$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" - fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . - - docker push "$CI_REGISTRY_IMAGE${tag}" - # Run this job in a branch where a Dockerfile exists + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + +docker-latest-no-db-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "true" + BASE_IMAGE: "node:18-bullseye-slim" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never @@ -40,60 +59,181 @@ docker-latest-build: - Dockerfile docker-arm32v7-build: - # Use the official docker image. image: docker:latest stage: build + variables: + EXCLUDE_DB: "false" + BASE_IMAGE: "arm32v7/node:18-bullseye-slim" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":arm32v7" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm32v7'" - else - tag=":arm32v7-$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" - fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm32v7 - - docker push "$CI_REGISTRY_IMAGE${tag}" - # Run this job in a branch where a Dockerfile exists + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.arm32v7 + - Dockerfile +docker-arm32v7-no-db-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "true" + BASE_IMAGE: "arm32v7/node:18-bullseye-slim" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + +docker-arm64v8-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "false" + BASE_IMAGE: "arm64v8/node:18-bullseye-slim" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + +docker-arm64v8-no-db-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "true" + BASE_IMAGE: "arm64v8/node:18-bullseye-slim" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile docker-nvidia-build: - # Use the official docker image. image: docker:latest stage: build + variables: + EXCLUDE_DB: "false" + BASE_IMAGE: "nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04" services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) script: - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag=":nvidia" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'nvidia'" - else - tag=":nvidia-$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" - fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia - - docker push "$CI_REGISTRY_IMAGE${tag}" - # Run this job in a branch where a Dockerfile exists + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" rules: - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" when: never - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.nvidia + - Dockerfile + +docker-nvidia-no-db-build: + image: docker:latest + stage: build + variables: + EXCLUDE_DB: "true" + BASE_IMAGE: "nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04" + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - | + BRANCH_PREFIX="$(if [ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]; then echo ${CI_COMMIT_REF_SLUG} ; else echo 'latest'; fi)" + NO_DB_SUFIX="$(${EXCLUDE_DB} && echo '-no-db' || echo '')" + BASE_IMAGE_ARCH="$(echo ${BASE_IMAGE} | cut -d'/' -f1)" + BASE_IMAGE_TYPE="$(echo ${BASE_IMAGE} | cut -d'/' -f2)" + ARCH_TYPE="$(if [ $BASE_IMAGE_ARCH != $BASE_IMAGE_TYPE ]; then echo '-'${BASE_IMAGE_ARCH}; else echo ''; fi )" + + export IMAGE_NAME="$CI_REGISTRY_IMAGE:${BRANCH_PREFIX}${ARCH_TYPE}${NO_DB_SUFIX}" + echo "Running on branch '${CI_COMMIT_BRANCH}', Image: ${IMAGE_NAME}" ; + + - docker build --pull --build-arg EXCLUDE_DB="${EXCLUDE_DB}" --build-arg BASE_IMAGE="${BASE_IMAGE}" -t "${IMAGE_NAME}" . -f "Dockerfile" + - docker push "${IMAGE_NAME}" + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile diff --git a/Docker/install_ffmpeg.sh b/Docker/install_ffmpeg.sh new file mode 100644 index 00000000..0732b99b --- /dev/null +++ b/Docker/install_ffmpeg.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +apt-get install -y \ + libfreetype6-dev \ + libgnutls28-dev \ + libmp3lame-dev \ + libass-dev \ + libogg-dev \ + libtheora-dev \ + libvorbis-dev \ + libvpx-dev \ + libwebp-dev \ + libssh2-1-dev \ + libopus-dev \ + librtmp-dev \ + libx264-dev \ + libx265-dev \ + x264 \ + ffmpeg + +ffmpeg -version \ No newline at end of file diff --git a/Docker/install_mariadb.sh b/Docker/install_mariadb.sh new file mode 100644 index 00000000..efb91341 --- /dev/null +++ b/Docker/install_mariadb.sh @@ -0,0 +1,19 @@ +#!/bin/sh +$EXCLUDE_DB > /dev/null + +if [ "$?" -ne 0 ] ; then + echo "Installing MariaDB" + set -ex + { \ + echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ + echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ + } | debconf-set-selections + + mkdir -p /var/lib/mysql + apt-get update + + apt-get install -y mariadb-server socat + + find /etc/mysql/ -name '*.cnf' -print0 | xargs -0 grep -lZE '^(bind-address|log)' | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/' + sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf +fi \ No newline at end of file diff --git a/Docker/install_nodejs.sh b/Docker/install_nodejs.sh new file mode 100644 index 00000000..7833b739 --- /dev/null +++ b/Docker/install_nodejs.sh @@ -0,0 +1,21 @@ +#!/bin/sh +which node > /dev/null + +if [ "$?" -ne 0 ] ; then + echo "Installing NodeJS 18" + mkdir -p /etc/apt/keyrings + apt-get install -y ca-certificates gnupg + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + + apt-get update -y --fix-missing + apt-get upgrade -y + apt-get install nodejs -y +fi + +node -v +npm -v + +npm i npm@latest pm2 pg -g --save +npm install --unsafe-perm diff --git a/Dockerfile b/Dockerfile index a94abea0..6157c72d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,8 @@ -FROM node:16-buster-slim +ARG BASE_IMAGE=node:18-bullseye-slim +FROM ${BASE_IMAGE} + +ARG DEBIAN_FRONTEND=noninteractive \ + EXCLUDE_DB=false ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -15,97 +19,57 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive + DB_DISABLE_INCLUDED=$EXCLUDE_DB -RUN mkdir -p /home/Shinobi /config /var/lib/mysql +WORKDIR /home/Shinobi +COPY . ./ -RUN apt update -y -RUN apt install wget curl net-tools -y +RUN apt-get update -y +RUN apt-get upgrade -y -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ +RUN apt-get install -y \ + wget \ + curl \ + net-tools \ + software-properties-common \ build-essential \ + git \ + python3 \ + sudo \ + pkg-config \ + apt-utils \ + yasm \ bzip2 \ coreutils \ procps \ gnutls-bin \ nasm \ tar \ - x264 + make \ + g++ \ + gcc \ + tar \ + xz-utils -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils +RUN sh /home/Shinobi/Docker/install_ffmpeg.sh +RUN sh /home/Shinobi/Docker/install_mariadb.sh +RUN sh /home/Shinobi/Docker/install_nodejs.sh - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins RUN chmod 777 /home/Shinobi -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ - -# Copy default configuration files +RUN chmod -R 777 /home/Shinobi/plugins RUN chmod -f +x /home/Shinobi/Docker/init.sh + RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi + +RUN apt-get update -y --fix-missing +RUN apt-get upgrade -y VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] EXPOSE 8080 443 21 25 -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] +ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] +CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] \ No newline at end of file diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 deleted file mode 100644 index 2e724f77..00000000 --- a/Dockerfile.arm32v7 +++ /dev/null @@ -1,109 +0,0 @@ -FROM arm32v7/node:16-buster-slim - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm && \ - apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y zip - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install pm2 -g && \ - npm install --unsafe-perm -COPY ./Docker/pm2.yml ./ - -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] - -EXPOSE 8080 - -ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "pm2.yml" ] diff --git a/Dockerfile.nodb b/Dockerfile.nodb deleted file mode 100644 index 8648484f..00000000 --- a/Dockerfile.nodb +++ /dev/null @@ -1,97 +0,0 @@ -FROM node:16-buster-slim - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=true -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y zip - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ - -RUN npm i pg --save - -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia deleted file mode 100644 index d6221971..00000000 --- a/Dockerfile.nvidia +++ /dev/null @@ -1,118 +0,0 @@ -FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu20.04 - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Additional packages to support NodeJS as the base build is NVidia CUDA -RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - -RUN apt install nodejs -y - -RUN node -v -RUN npm -v - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ - -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/definitions/base.js b/definitions/base.js index 80f6498b..136b50b1 100644 --- a/definitions/base.js +++ b/definitions/base.js @@ -748,8 +748,7 @@ module.exports = function(s,config,lang){ "name": "detail=stream_type", "field": lang["Stream Type"], "description": lang["fieldTextStreamType"], - "default": "mp4", - "example": "", + "default": "hls", "selector": "h_st", "fieldType": "select", "attribute": `triggerChange="#add_monitor [detail=stream_vcodec]" triggerChangeIgnore="b64,mjpeg,jpeg,gif"`, diff --git a/languages/en_CA.json b/languages/en_CA.json index f4b4e075..e1923e31 100644 --- a/languages/en_CA.json +++ b/languages/en_CA.json @@ -145,6 +145,7 @@ "Input Settings": "Input Settings", "Connection": "Connection", "Video Set": "Video Set", + "Video Accessed": "Video Accessed", "notEnoughFramesText1": "Not Enough Frames for compilation.", "Allow API Trigger": "Allow API Trigger", "When Detector is Off": "When Detector is Off", @@ -480,6 +481,7 @@ "Database": "Database", "Database Not Found": "Database Not Found", "User Not Found": "User Not Found", + "User Logged Out": "User Logged Out", "Not Found": "Not Found", "Save Links to Database": "Save Links to Database", "Uploaders": "Uploaders", diff --git a/libs/dataPort.js b/libs/dataPort.js index a20c5143..3854826e 100644 --- a/libs/dataPort.js +++ b/libs/dataPort.js @@ -39,6 +39,8 @@ module.exports = function(s,config,lang,app,io){ s.debugLog(data.data) break; default: + console.log(`No Data Port Handler!`) + console.log(`here's what we got :`) console.log(data) break; } diff --git a/libs/events/utils.js b/libs/events/utils.js index 98e50cdb..004d6950 100644 --- a/libs/events/utils.js +++ b/libs/events/utils.js @@ -448,8 +448,8 @@ module.exports = (s,config,lang) => { createEventBasedRecording(d,moment(eventTime).subtract(secondBefore,'seconds').format('YYYY-MM-DDTHH-mm-ss')) } d.currentTime = eventTime - d.currentTimestamp = s.timeObject(d.currentTime).format() - d.screenshotName = eventDetails.reason + '_'+(monitorConfig.name.replace(/[^\w\s]/gi,''))+'_'+d.id+'_'+d.ke+'_'+s.formattedTime() + d.currentTimestamp = s.timeObject(eventTime).format() + d.screenshotName = eventDetails.reason + '_'+(monitorConfig.name.replace(/[^\w\s]/gi,''))+'_'+d.id+'_'+d.ke+'_'+s.formattedTime(eventTime) d.screenshotBuffer = null if(filter.webhook && monitorDetails.detector_webhook === '1' && !s.group[d.ke].activeMonitors[d.id].detector_webhook){ @@ -836,10 +836,11 @@ module.exports = (s,config,lang) => { function getObjectTagsFromMatrices(d){ if(d.details.reason === 'motion'){ return [getTagWithIcon(lang.Motion)] - }else{ + }else if(d.details.matrices){ const matrices = d.details.matrices return [...new Set(matrices.map(matrix => getTagWithIcon(matrix.tag)))]; } + return [getTagWithIcon(d.details.reason)] } function getObjectTagNotifyText(d){ const monitorId = d.mid || d.id diff --git a/libs/extenders.js b/libs/extenders.js index 61b66361..7cf3595d 100644 --- a/libs/extenders.js +++ b/libs/extenders.js @@ -25,6 +25,8 @@ module.exports = function(s,config){ createExtension(`beforeAccountSave`) createExtension(`onTwoFactorAuthCodeNotification`) createExtension(`onStalePurgeLock`) + createExtension(`onVideoAccess`) + createExtension(`onLogout`) ////// EVENTS ////// createExtension(`onEventTrigger`) createExtension(`onEventTriggerBeforeFilter`) @@ -62,6 +64,7 @@ module.exports = function(s,config){ createExtension(`onEventBasedRecordingComplete`) createExtension(`onEventBasedRecordingStart`) createExtension(`onBeforeInsertCompletedVideo`) + createExtension(`onCloudVideoUploaded`) /////// TIMELAPSE //////// createExtension(`onInsertTimelapseFrame`) } diff --git a/libs/ffmpeg/builders.js b/libs/ffmpeg/builders.js index 727ee359..eca1fe1c 100644 --- a/libs/ffmpeg/builders.js +++ b/libs/ffmpeg/builders.js @@ -548,7 +548,7 @@ module.exports = (s,config,lang) => { recordingFilters.push(`fps=${videoFps}`) } } - if(videoExtIsMp4){ + if(videoExtIsMp4 && !config.noDefaultRecordingSegmentFormatOptions){ customRecordingFlags.push(`-segment_format_options movflags=faststart`) } if(videoCodec === 'h264_vaapi'){ @@ -617,7 +617,8 @@ module.exports = (s,config,lang) => { const sendFramesGlobally = (e.details.detector_send_frames === '1') const objectDetectorOutputIsEnabled = (e.details.detector_use_detect_object === '1') const builtInMotionDetectorIsEnabled = (e.details.detector_pam === '1') - const sendFramesToObjectDetector = (e.details.detector_send_frames_object !== '0' && e.details.detector_use_detect_object === '1') + const objectDetectorSendFrames = e.details.detector_send_frames_object !== '0' || e.details.detector_pam !== '1' + const sendFramesToObjectDetector = (objectDetectorSendFrames && e.details.detector_use_detect_object === '1') const baseWidth = e.details.detector_scale_x ? e.details.detector_scale_x : '640' const baseHeight = e.details.detector_scale_y ? e.details.detector_scale_y : '480' const baseDimensionsFlag = `-s ${baseWidth}x${baseHeight}` diff --git a/libs/ffmpeg/possibleWarnings.js b/libs/ffmpeg/possibleWarnings.js index 8f4097bc..d271743b 100644 --- a/libs/ffmpeg/possibleWarnings.js +++ b/libs/ffmpeg/possibleWarnings.js @@ -1,5 +1,6 @@ module.exports = (monitor,probeResult,config,lang) => { const primaryVideoStream = probeResult.video[0] + const noPortOr554 = !monitor.port || monitor.port === 554 return [ { isTrue: monitor.details.stream_vcodec === 'copy' && primaryVideoStream.codec === 'hevc', @@ -83,7 +84,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && monitor.protocol === 'rtmp' ), title: lang['Automatic Field Fill'], @@ -95,7 +96,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && monitor.protocol === 'http' ), title: lang['Automatic Field Fill'], @@ -107,7 +108,7 @@ module.exports = (monitor,probeResult,config,lang) => { }, { isTrue: ( - !monitor.port && + noPortOr554 && (monitor.protocol === 'rtmps' || monitor.protocol === 'https') ), diff --git a/libs/ffmpeg/utils.js b/libs/ffmpeg/utils.js index e20402ec..e678051b 100644 --- a/libs/ffmpeg/utils.js +++ b/libs/ffmpeg/utils.js @@ -367,6 +367,18 @@ Run "npm install ffbinaries" to get this static FFmpeg downloader.` } return response } + async function getWarningChangesForMonitor(monitorConfig){ + const probeResponse = await probeMonitor(monitorConfig,2000,true) + const probeStreams = getStreamInfoFromProbe(probeResponse.result) + const warnings = createWarningsForConfiguration(monitorConfig,probeStreams) + const configPartial = warnings.length > 0 ? buildMonitorConfigPartialFromWarnings(warnings) : {} + return { + configPartial, + warnings, + probeResponse, + probeStreams, + } + } return { ffprobe: runFFprobe, probeMonitor: probeMonitor, @@ -386,5 +398,6 @@ Run "npm install ffbinaries" to get this static FFmpeg downloader.` checkStaticBuilds: checkStaticBuilds, checkVersion: checkVersion, checkHwAccelMethods: checkHwAccelMethods, + getWarningChangesForMonitor, } } diff --git a/libs/monitor.js b/libs/monitor.js index 4a366722..95506de6 100644 --- a/libs/monitor.js +++ b/libs/monitor.js @@ -13,6 +13,8 @@ module.exports = function(s,config,lang){ } = require('./basic/utils.js')(process.cwd(),config) const { splitForFFMPEG, + applyPartialToConfiguration, + getWarningChangesForMonitor, } = require('./ffmpeg/utils.js')(s,config,lang) const { processKill, @@ -551,7 +553,6 @@ module.exports = function(s,config,lang){ return } form.mid = form.mid.replace(/[^\w\s]/gi,'').replace(/ /g,'') - form = s.cleanMonitorObjectForDatabase(form) const selectResponse = await s.knexQueryPromise({ action: "select", columns: "*", @@ -570,6 +571,16 @@ module.exports = function(s,config,lang){ ke: form.ke, mon: form } + // auto correct + const { + configPartial, + warnings, + probeResponse, + probeStreams, + } = await getWarningChangesForMonitor(form) + applyPartialToConfiguration(form,configPartial) + form = s.cleanMonitorObjectForDatabase(form) + // if(monitorExists){ txData.new = false Object.keys(form).forEach(function(v){ diff --git a/libs/monitor/utils.js b/libs/monitor/utils.js index 5db663da..00cd7ddf 100644 --- a/libs/monitor/utils.js +++ b/libs/monitor/utils.js @@ -10,11 +10,8 @@ const SoundDetection = require('shinobi-sound-detection') const streamViewerCountTimeouts = {} module.exports = (s,config,lang) => { const { - probeMonitor, - getStreamInfoFromProbe, applyPartialToConfiguration, - createWarningsForConfiguration, - buildMonitorConfigPartialFromWarnings, + getWarningChangesForMonitor, createPipeArray, splitForFFMPEG, sanitizedFfmpegCommand, @@ -1353,6 +1350,7 @@ module.exports = (s,config,lang) => { // doFatalErrorCatch(e,d) // },15000) break; + case checkLog(d,'Could not find codec parameters'): case checkLog(d,'No route to host'): activeMonitor.timeoutToRestart = setTimeout(async () => { doFatalErrorCatch(e,d) @@ -1662,17 +1660,18 @@ module.exports = (s,config,lang) => { return; } if(config.probeMonitorOnStart === true){ - const probeResponse = await probeMonitor(monitorConfig,2000,true) - const probeStreams = getStreamInfoFromProbe(probeResponse.result) - activeMonitor.probeResult = probeStreams - const warnings = createWarningsForConfiguration(monitorConfig,probeStreams) - activeMonitor.warnings = warnings + const { + configPartial, + warnings, + probeResponse, + probeStreams, + } = await getWarningChangesForMonitor(monitorConfig) if(warnings.length > 0){ - const configPartial = buildMonitorConfigPartialFromWarnings(warnings) applyPartialToConfiguration(e,configPartial) applyPartialToConfiguration(activeMonitor,configPartial) applyPartialToConfiguration(s.group[groupKey].rawMonitorConfigurations[monitorId],configPartial) } + activeMonitor.warnings = warnings } activeMonitor.isStarted = true if(e.details && e.details.dir && e.details.dir !== ''){ @@ -1694,6 +1693,7 @@ module.exports = (s,config,lang) => { } try{ await launchMonitorProcesses(e) + resetStreamCheck(e) }catch(err){ console.error(err) } @@ -1748,9 +1748,13 @@ module.exports = (s,config,lang) => { } function isGroupBelowMaxMonitorCount(groupKey){ const theGroup = s.group[groupKey]; - const initData = theGroup.init; - const maxCamerasAllowed = parseInt(initData.max_camera) || false; - return (!maxCamerasAllowed || Object.keys(theGroup.activeMonitors).length <= parseInt(maxCamerasAllowed)) + try{ + const initData = theGroup.init; + const maxCamerasAllowed = parseInt(initData.max_camera) || false; + return (!maxCamerasAllowed || Object.keys(theGroup.activeMonitors).length <= parseInt(maxCamerasAllowed)) + }catch(err){ + return true + } } function getStreamDirectory(options){ const streamDir = s.dir.streams + options.ke + '/' + options.mid + '/' diff --git a/libs/notifications/webhook.js b/libs/notifications/webhook.js index 194546c3..b133deef 100644 --- a/libs/notifications/webhook.js +++ b/libs/notifications/webhook.js @@ -240,7 +240,29 @@ module.exports = function(s,config,lang,getSnapshot){ "placeholder": "http://your-webhook-point/onEvent/{{INNER_EVENT_TITLE}}?info={{INNER_EVENT_INFO}}", "field": lang["Webhook URL"], "form-group-class":"u_global_webhook_input u_global_webhook_1", - } + }, + { + hidden: true, + "name": "detail=global_webhook_method", + "field": lang['Call Method'], + "default": "GET", + "form-group-class":"u_global_webhook_input u_global_webhook_1", + "fieldType": "select", + "possible": [ + { + "name": `GET (${lang.Default})`, + "value": "GET" + }, + { + "name": "PUT", + "value": "PUT" + }, + { + "name": "POST", + "value": "POST" + } + ] + }, ] } s.definitions["Event Filters"].blocks["Action for Selected"].info.push({ diff --git a/libs/uploaders/amazonS3.js b/libs/uploaders/amazonS3.js index c9f02149..95dd6497 100644 --- a/libs/uploaders/amazonS3.js +++ b/libs/uploaders/amazonS3.js @@ -315,79 +315,123 @@ module.exports = function(s,config,lang){ "default": "", "example": "", "possible": [ - { - "name": "US West (N. California)", - "value": "us-west-1" - }, - { - "name": "US West (Oregon)", - "value": "us-west-2" - }, - { - "name": "US East (Ohio)", - "value": "us-east-2" - }, - { - "name": "US East (N. Virginia)", - "value": "us-east-1" - }, - { - "name": "Asia Pacific (Mumbai)", - "value": "ap-south-1" - }, - { - "name": "Asia Pacific (Seoul)", - "value": "ap-northeast-2" - }, - { - "name": "Asia Pacific (Osaka-Local)**", - "value": "ap-northeast-3" - }, - { - "name": "Asia Pacific (Singapore)", - "value": "ap-southeast-1" - }, - { - "name": "Asia Pacific (Sydney)", - "value": "ap-southeast-2" - }, - { - "name": "Asia Pacific (Tokyo)", - "value": "ap-northeast-1" - }, - { - "name": "Canada (Central)", - "value": "ca-central-1" - }, - { - "name": "China (Beijing)", - "value": "cn-north-1" - }, - { - "name": "China (Ningxia)", - "value": "cn-northwest-1" - }, - { - "name": "EU (Frankfurt)", - "value": "eu-central-1" - }, - { - "name": "EU (Ireland)", - "value": "eu-west-1" - }, - { - "name": "EU (London)", - "value": "eu-west-2" - }, - { - "name": "EU (Paris)", - "value": "eu-west-3" - }, - { - "name": "South America (São Paulo)", - "value": "sa-east-1" - } - ] + { + "name": "US West (N. California)", + "value": "us-west-1" + }, + { + "name": "US West (Oregon)", + "value": "us-west-2" + }, + { + "name": "US East (Ohio)", + "value": "us-east-2" + }, + { + "name": "US East (N. Virginia)", + "value": "us-east-1" + }, + { + "name": "Canada (Central)", + "value": "ca-central-1" + }, + { + "name": "South America (São Paulo)", + "value": "sa-east-1" + }, + { + "name": "EU (Frankfurt)", + "value": "eu-central-1" + }, + { + "name": "EU (Ireland)", + "value": "eu-west-1" + }, + { + "name": "EU (London)", + "value": "eu-west-2" + }, + { + "name": "EU (Paris)", + "value": "eu-west-3" + }, + { + "name": "Europe (Milan)", + "value": "eu-south-1" + }, + { + "name": "Europe (Spain)", + "value": "eu-south-2" + }, + { + "name": "Europe (Zurich)", + "value": "eu-central-2" + }, + { + "name": "Asia Pacific (Mumbai)", + "value": "ap-south-1" + }, + { + "name": "Asia Pacific (Seoul)", + "value": "ap-northeast-2" + }, + { + "name": "Asia Pacific (Osaka-Local)**", + "value": "ap-northeast-3" + }, + { + "name": "Asia Pacific (Singapore)", + "value": "ap-southeast-1" + }, + { + "name": "Asia Pacific (Sydney)", + "value": "ap-southeast-2" + }, + { + "name": "Asia Pacific (Tokyo)", + "value": "ap-northeast-1" + }, + { + "name": "Asia Pacific (Hong Kong)", + "value": "ap-east-1" + }, + { + "name": "Asia Pacific (Hyderabad)", + "value": "ap-south-2" + }, + { + "name": "Asia Pacific (Jakarta)", + "value": "ap-southeast-3" + }, + { + "name": "Asia Pacific (Melbourne)", + "value": "ap-southeast-4" + }, + { + "name": "China (Beijing)", + "value": "cn-north-1" + }, + { + "name": "China (Ningxia)", + "value": "cn-northwest-1" + }, + { + "name": "Africa (Cape Town)", + "value": "af-south-1" + }, + { + "name": "Middle East (Bahrain)", + "value": "me-south-1" + }, + { + "name": "Middle East (UAE)", + "value": "me-central-1" + }, + { + "name": "il-central-1", + "value": "il-central-1" + } + ] }, { "hidden": true, diff --git a/libs/uploaders/backblazeB2.js b/libs/uploaders/backblazeB2.js index c28b0d90..b06294fb 100644 --- a/libs/uploaders/backblazeB2.js +++ b/libs/uploaders/backblazeB2.js @@ -3,18 +3,19 @@ const { Readable } = require('stream'); const B2 = require('backblaze-b2') module.exports = function(s,config,lang){ //Backblaze B2 + var serviceProvider = 'b2' var beforeAccountSaveForBackblazeB2 = function(d){ //d = save event d.formDetails.b2_use_global=d.d.b2_use_global d.formDetails.use_bb_b2=d.d.use_bb_b2 } var cloudDiskUseStartupForBackblazeB2 = function(group,userDetails){ - group.cloudDiskUse['b2'].name = 'Backblaze B2' - group.cloudDiskUse['b2'].sizeLimitCheck = (userDetails.use_bb_b2_size_limit === '1') + group.cloudDiskUse[serviceProvider].name = 'Backblaze B2' + group.cloudDiskUse[serviceProvider].sizeLimitCheck = (userDetails.use_bb_b2_size_limit === '1') if(!userDetails.bb_b2_size_limit || userDetails.bb_b2_size_limit === ''){ - group.cloudDiskUse['b2'].sizeLimit = 10000 + group.cloudDiskUse[serviceProvider].sizeLimit = 10000 }else{ - group.cloudDiskUse['b2'].sizeLimit = parseFloat(userDetails.bb_b2_size_limit) + group.cloudDiskUse[serviceProvider].sizeLimit = parseFloat(userDetails.bb_b2_size_limit) } } var loadBackblazeB2ForUser = function(e){ @@ -83,7 +84,7 @@ module.exports = function(s,config,lang){ }catch(err){ var videoDetails = video.details } - if(video.type !== 'b2'){ + if(video.type !== serviceProvider){ callback() return } @@ -128,30 +129,35 @@ module.exports = function(s,config,lang){ }).then(function(resp){ const uploadResponse = resp.data if(theGroup.init.bb_b2_log === '1' && uploadResponse.fileId){ + const insertDetails = { + bucketId : uploadResponse.bucketId, + fileId : uploadResponse.fileId, + fileName : uploadResponse.fileName + } + const insertQuery = { + mid: e.mid, + ke: e.ke, + time: k.startTime, + status: 1, + type : serviceProvider, + details: insertDetails, + size: k.filesize, + end: k.endTime, + href: '' + } s.knexQuery({ action: "insert", table: "Cloud Videos", - insert: { - mid: e.mid, - ke: e.ke, - time: k.startTime, - status: 1, - type : 'b2', - details: s.s({ - bucketId : uploadResponse.bucketId, - fileId : uploadResponse.fileId, - fileName : uploadResponse.fileName - }), - size: k.filesize, - end: k.endTime, - href: '' - } + insert: Object.assign({},insertQuery,{details: s.s(insertDetails)}) }) s.setCloudDiskUsedForGroup(e.ke,{ amount : k.filesizeMB, - storageType : 'b2' + storageType : serviceProvider + }) + s.purgeCloudDiskForGroup(e,serviceProvider) + s.onCloudVideoUploadedExtensions.forEach((extender) => { + extender(insertQuery) }) - s.purgeCloudDiskForGroup(e,'b2') } }).catch(backblazeErr) }) @@ -183,7 +189,7 @@ module.exports = function(s,config,lang){ } //backblaze b2 s.addCloudUploader({ - name: 'b2', + name: serviceProvider, loadGroupAppExtender: loadBackblazeB2ForUser, unloadGroupAppExtender: unloadBackblazeB2ForUser, insertCompletedVideoExtender: uploadVideoToBackblazeB2, diff --git a/libs/uploaders/s3based.js b/libs/uploaders/s3based.js index 34c053be..7894e08c 100644 --- a/libs/uploaders/s3based.js +++ b/libs/uploaders/s3based.js @@ -324,84 +324,128 @@ module.exports = function(s,config,lang){ "description": "", "default": "", "example": "", - "possible": [ - { - "name": lang['No Region'], - "value": "" - }, - { - "name": "US West 1", - "value": "us-west-1" - }, - { - "name": "US West 2", - "value": "us-west-2" - }, - { - "name": "US East 1", - "value": "us-east-1" - }, - { - "name": "US East 2", - "value": "us-east-2" - }, - { - "name": "Asia Pacific 1", - "value": "ap-south-1" - }, - { - "name": "Asia Pacific 2", - "value": "ap-northeast-2" - }, - { - "name": "Asia Pacific 3", - "value": "ap-northeast-3" - }, - { - "name": "Asia Pacific 4", - "value": "ap-southeast-1" - }, - { - "name": "Asia Pacific 5", - "value": "ap-southeast-2" - }, - { - "name": "Asia Pacific 6", - "value": "ap-northeast-1" - }, - { - "name": "Canada 1", - "value": "ca-central-1" - }, - { - "name": "China 1", - "value": "cn-north-1" - }, - { - "name": "China 1", - "value": "cn-northwest-1" - }, - { - "name": "EU 1", - "value": "eu-central-1" - }, - { - "name": "EU 2", - "value": "eu-west-1" - }, - { - "name": "EU 3", - "value": "eu-west-2" - }, - { - "name": "EU 4", - "value": "eu-west-3" - }, - { - "name": "South America 1", - "value": "sa-east-1" - } - ] + "possible": [ + { + "name": lang['No Region'], + "value": "" + }, + { + "name": "US West (N. California)", + "value": "us-west-1" + }, + { + "name": "US West (Oregon)", + "value": "us-west-2" + }, + { + "name": "US East (Ohio)", + "value": "us-east-2" + }, + { + "name": "US East (N. Virginia)", + "value": "us-east-1" + }, + { + "name": "Canada (Central)", + "value": "ca-central-1" + }, + { + "name": "South America (São Paulo)", + "value": "sa-east-1" + }, + { + "name": "EU (Frankfurt)", + "value": "eu-central-1" + }, + { + "name": "EU (Ireland)", + "value": "eu-west-1" + }, + { + "name": "EU (London)", + "value": "eu-west-2" + }, + { + "name": "EU (Paris)", + "value": "eu-west-3" + }, + { + "name": "Europe (Milan)", + "value": "eu-south-1" + }, + { + "name": "Europe (Spain)", + "value": "eu-south-2" + }, + { + "name": "Europe (Zurich)", + "value": "eu-central-2" + }, + { + "name": "Asia Pacific (Mumbai)", + "value": "ap-south-1" + }, + { + "name": "Asia Pacific (Seoul)", + "value": "ap-northeast-2" + }, + { + "name": "Asia Pacific (Osaka-Local)**", + "value": "ap-northeast-3" + }, + { + "name": "Asia Pacific (Singapore)", + "value": "ap-southeast-1" + }, + { + "name": "Asia Pacific (Sydney)", + "value": "ap-southeast-2" + }, + { + "name": "Asia Pacific (Tokyo)", + "value": "ap-northeast-1" + }, + { + "name": "Asia Pacific (Hong Kong)", + "value": "ap-east-1" + }, + { + "name": "Asia Pacific (Hyderabad)", + "value": "ap-south-2" + }, + { + "name": "Asia Pacific (Jakarta)", + "value": "ap-southeast-3" + }, + { + "name": "Asia Pacific (Melbourne)", + "value": "ap-southeast-4" + }, + { + "name": "China (Beijing)", + "value": "cn-north-1" + }, + { + "name": "China (Ningxia)", + "value": "cn-northwest-1" + }, + { + "name": "Africa (Cape Town)", + "value": "af-south-1" + }, + { + "name": "Middle East (Bahrain)", + "value": "me-south-1" + }, + { + "name": "Middle East (UAE)", + "value": "me-central-1" + }, + { + "name": "il-central-1", + "value": "il-central-1" + } + ] }, { "hidden": true, diff --git a/libs/user.js b/libs/user.js index ea580253..3187c250 100644 --- a/libs/user.js +++ b/libs/user.js @@ -18,6 +18,7 @@ module.exports = function(s,config,lang){ deleteCloudTimelapseFrames, resetAllStorageCounters, } = require("./user/utils.js")(s,config,lang); + require("./user/logger.js")(s,config,lang) let purgeDiskGroup = () => {} const runQuery = async.queue(function(groupKey, callback) { purgeDiskGroup(groupKey,callback) diff --git a/libs/user/logger.js b/libs/user/logger.js new file mode 100644 index 00000000..f26c25a6 --- /dev/null +++ b/libs/user/logger.js @@ -0,0 +1,31 @@ +module.exports = function(s,config,lang){ + s.onVideoAccess((videoRow,user,groupKey,monitorId,ip) => { + s.userLog({ + ke: groupKey, + mid: '$USER', + },{ + type: lang['Video Accessed'], + msg: { + user: { + mail: user.mail, + uid: user.uid, + ip, + }, + video: videoRow + } + }) + }) + s.onLogout((user,groupKey,userId,ip) => { + s.userLog({ + ke: groupKey, + mid: '$USER', + },{ + type: lang['User Logged Out'], + msg: { + mail: user.mail, + uid: user.uid, + ip, + } + }) + }) +} diff --git a/libs/webServerPaths.js b/libs/webServerPaths.js index 104fa3b3..25db79b2 100644 --- a/libs/webServerPaths.js +++ b/libs/webServerPaths.js @@ -107,9 +107,18 @@ module.exports = function(s,config,lang,app,io){ * API : Logout */ app.get(config.webPaths.apiPrefix+':auth/logout/:ke/:id', function (req,res){ - if(s.group[req.params.ke] && s.group[req.params.ke].users[req.params.auth] && s.group[req.params.ke].users[req.params.auth].details){ - delete(s.api[req.params.auth]); - delete(s.group[req.params.ke].users[req.params.auth]); + const groupKey = req.params.ke + const userId = req.params.id + const authToken = req.params.auth + const user = s.group[groupKey] && s.group[groupKey].users[authToken] && s.group[groupKey].users[authToken].details ? s.group[groupKey].users[authToken] : null; + if(user){ + const clientIp = s.getClientIp(req) + const user = s.group[groupKey].users[authToken] + s.onLogoutExtensions.forEach((extender) => { + extender(user,groupKey,userId,clientIp) + }); + delete(s.api[authToken]); + delete(s.group[groupKey].users[authToken]); s.knexQuery({ action: "update", table: "Users", @@ -117,9 +126,9 @@ module.exports = function(s,config,lang,app,io){ auth: '', }, where: [ - ['auth','=',req.params.auth], - ['ke','=',req.params.ke], - ['uid','=',req.params.id], + ['auth','=',authToken], + ['ke','=',groupKey], + ['uid','=',userId], ] }) res.end(s.prettyPrint({ok:true,msg:'You have been logged out, session key is now inactive.'})) @@ -1454,6 +1463,10 @@ module.exports = function(s,config,lang,app,io){ }else{ s.streamMp4FileOverHttp(filePath,req,res,!!req.query.pureStream) } + const clientIp = s.getClientIp(req) + s.onVideoAccessExtensions.forEach((extender) => { + extender(videoRow,user,groupKey,monitorId,clientIp) + }) }else{ s.closeJsonResponse(res,{ ok: false, diff --git a/libs/webServerStreamPaths.js b/libs/webServerStreamPaths.js index cd2a85f5..1d7b8478 100644 --- a/libs/webServerStreamPaths.js +++ b/libs/webServerStreamPaths.js @@ -159,7 +159,7 @@ module.exports = function(s,config,lang,app){ return; } s.checkChildProxy(req.params,function(){ - if(s.group[req.params.ke]&&s.group[req.params.ke].activeMonitors[req.params.id]){ + if(s.group[req.params.ke]&&s.group[req.params.ke].activeMonitors&&s.group[req.params.ke].activeMonitors[req.params.id]){ if(user.permissions.watch_stream==="0"||user.details.sub&&user.details.allmonitors!=='1'&&user.details.monitors.indexOf(req.params.id)===-1){ res.end(user.lang['Not Permitted']) return diff --git a/web/assets/css/dashboard.css b/web/assets/css/dashboard.css index e007b649..7a61b7fd 100644 --- a/web/assets/css/dashboard.css +++ b/web/assets/css/dashboard.css @@ -431,3 +431,8 @@ ul.squeeze { flex-direction: column; height: 100vh; } + +.log-item .msg-tree > ul { + list-style: none; + padding-left: 0; +} diff --git a/web/assets/js/bs5.dashboard-base.js b/web/assets/js/bs5.dashboard-base.js index 0edb4b92..a8d7cc1c 100644 --- a/web/assets/js/bs5.dashboard-base.js +++ b/web/assets/js/bs5.dashboard-base.js @@ -327,11 +327,13 @@ function compileConnectUrl(options){ function jsonToHtmlBlock(target){ var html = '' if(target instanceof Object){ + var html = '' }else{ html += `${target}` } @@ -836,7 +838,7 @@ function buildLogRow(v){ ${humanMonitorName}${v.info && v.info.type ? v.info.type : v.mid}
-
${jsonToHtmlBlock(v.info.msg)}
+
${jsonToHtmlBlock(v.info.msg)}