From 2fe9cd1831ab9bf3b44b3abdf4aa5adcfe0e42f4 Mon Sep 17 00:00:00 2001 From: Elad Bar Date: Sun, 24 Sep 2023 10:13:19 +0300 Subject: [PATCH] docker images upgrade --- .gitlab-ci.yml | 87 ++++++++++++++++++++++++++++ Docker/install_dependencies.sh | 85 +++++++++++++++++++++++++++ Dockerfile | 91 +++-------------------------- Dockerfile.arm32v7 | 91 +++-------------------------- Dockerfile.nodb | 97 ------------------------------- Dockerfile.nvidia | 101 ++++----------------------------- 6 files changed, 200 insertions(+), 352 deletions(-) create mode 100644 Docker/install_dependencies.sh delete mode 100644 Dockerfile.nodb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79d095a1..ff37bb9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,6 +39,35 @@ docker-latest-build: exists: - Dockerfile +docker-latest-no-db-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag="latest-no-db" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest-no-db'" + else + tag="latest-no-db:$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile + docker-arm32v7-build: # Use the official docker image. image: docker:latest @@ -68,6 +97,35 @@ docker-arm32v7-build: exists: - Dockerfile.arm32v7 +docker-arm32v7-no-db-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag=":arm32v7-no-db" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'arm32v7-no-db'" + else + tag=":arm32v7-no-db-$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.arm32v7 + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile.arm32v7 + docker-nvidia-build: # Use the official docker image. @@ -97,3 +155,32 @@ docker-nvidia-build: - if: $CI_COMMIT_BRANCH exists: - Dockerfile.nvidia + +docker-nvidia-no-db-build: + # Use the official docker image. + image: docker:latest + stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) + script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag=":nvidia-no-db" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'nvidia-no-db'" + else + tag=":nvidia-no-db-$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull --build-arg VAR_EXCLUDE_DB=true -t "$CI_REGISTRY_IMAGE${tag}" . -f Dockerfile.nvidia + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH != "main" && $CI_COMMIT_BRANCH != "dev" + when: never + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile.nvidia diff --git a/Docker/install_dependencies.sh b/Docker/install_dependencies.sh new file mode 100644 index 00000000..de213126 --- /dev/null +++ b/Docker/install_dependencies.sh @@ -0,0 +1,85 @@ +#!/bin/sh +mkdir -p /etc/apt/keyrings + +apt update -y --fix-missing +apt upgrade -y +apt update -y --fix-missing + +apt install -y \ + wget \ + curl \ + net-tools \ + software-properties-common \ + libfreetype6-dev \ + libgnutls28-dev \ + libmp3lame-dev \ + libass-dev \ + libogg-dev \ + libtheora-dev \ + libvorbis-dev \ + libvpx-dev \ + libwebp-dev \ + libssh2-1-dev \ + libopus-dev \ + librtmp-dev \ + libx264-dev \ + libx265-dev \ + yasm \ + build-essential \ + bzip2 \ + coreutils \ + procps \ + gnutls-bin \ + nasm \ + tar \ + x264 \ + ffmpeg \ + git \ + make \ + g++ \ + gcc \ + pkg-config \ + python3 \ + tar \ + sudo \ + xz-utils \ + ca-certificates \ + gnupg + +if [ "$INSTALL_NODE" = "true" ] ; then set -ex; \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg ; \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list ; \ + apt update -y --fix-missing ; \ + apt upgrade -y ; \ + apt install nodejs -y ; fi + +node -v +npm -v + +npm i npm@latest pm2 pg -g --save +npm install --unsafe-perm + +chmod 777 /home/Shinobi +chmod -R 777 /home/Shinobi/plugins +chmod -f +x /home/Shinobi/Docker/init.sh + +sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh + +ffmpeg -version + +# Install MariaDB server... the debian way +if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ + { \ + echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ + echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ + } | debconf-set-selections; \ + mkdir -p /var/lib/mysql; \ + apt-get update; \ + apt-get install -y \ + "mariadb-server" \ + socat \ + ; \ + find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; \ + sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi diff --git a/Dockerfile b/Dockerfile index a94abea0..4414a790 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -FROM node:16-buster-slim +FROM node:18-bullseye-slim + +ARG DEBIAN_FRONTEND=noninteractive \ + VAR_EXCLUDE_DB=false ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -15,97 +18,19 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - + DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN chmod 777 /home/Shinobi -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ +COPY . ./ -# Copy default configuration files -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi +RUN sh /home/Shinobi/Docker/install_dependencies.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] EXPOSE 8080 443 21 25 -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] +ENTRYPOINT ["sh", "/home/Shinobi/Docker/init.sh"] CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 index 2e724f77..9957f392 100644 --- a/Dockerfile.arm32v7 +++ b/Dockerfile.arm32v7 @@ -1,4 +1,7 @@ -FROM arm32v7/node:16-buster-slim +FROM arm32v7/node:18-bullseye-slim + +ARG DEBIAN_FRONTEND=noninteractive \ + VAR_EXCLUDE_DB=false ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -15,95 +18,19 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm && \ - apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y zip - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - + DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install pm2 -g && \ - npm install --unsafe-perm -COPY ./Docker/pm2.yml ./ +COPY . ./ -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh +RUN sh /home/Shinobi/Docker/install_dependencies.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] -EXPOSE 8080 +EXPOSE 8080 443 21 25 ENTRYPOINT ["/home/Shinobi/Docker/init.sh"] -CMD [ "pm2-docker", "pm2.yml" ] +CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.nodb b/Dockerfile.nodb deleted file mode 100644 index 8648484f..00000000 --- a/Dockerfile.nodb +++ /dev/null @@ -1,97 +0,0 @@ -FROM node:16-buster-slim - -ENV DB_USER=majesticflame \ - DB_PASSWORD='' \ - DB_HOST='localhost' \ - DB_DATABASE=ccio \ - DB_PORT=3306 \ - DB_TYPE='mysql' \ - SUBSCRIPTION_ID=sub_XXXXXXXXXXXX \ - PLUGIN_KEYS='{}' \ - SSL_ENABLED='false' \ - SSL_COUNTRY='CA' \ - SSL_STATE='BC' \ - SSL_LOCATION='Vancouver' \ - SSL_ORGANIZATION='Shinobi Systems' \ - SSL_ORGANIZATION_UNIT='IT Department' \ - SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=true -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y zip - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - - -WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ - -RUN npm i pg --save - -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi - -VOLUME ["/home/Shinobi/videos"] -VOLUME ["/home/Shinobi/libs/customAutoLoad"] -VOLUME ["/config"] - -EXPOSE 8080 443 21 25 - -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] - -CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ] diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia index d6221971..886b92b9 100644 --- a/Dockerfile.nvidia +++ b/Dockerfile.nvidia @@ -1,4 +1,8 @@ -FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu20.04 +FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04 + +ARG DEBIAN_FRONTEND=noninteractive \ + VAR_EXCLUDE_DB=false \ + VAR_INSTALL_NODE=true ENV DB_USER=majesticflame \ DB_PASSWORD='' \ @@ -15,104 +19,21 @@ ENV DB_USER=majesticflame \ SSL_ORGANIZATION='Shinobi Systems' \ SSL_ORGANIZATION_UNIT='IT Department' \ SSL_COMMON_NAME='nvr.ninja' \ - DB_DISABLE_INCLUDED=false -ARG DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /home/Shinobi /config /var/lib/mysql - -RUN apt update -y -RUN apt install wget curl net-tools -y - -# Additional packages to support NodeJS as the base build is NVidia CUDA -RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - -RUN apt install nodejs -y - -RUN node -v -RUN npm -v - -# Install MariaDB server... the debian way -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then set -ex; \ - { \ - echo "mariadb-server" mysql-server/root_password password '${DB_ROOT_PASSWORD}'; \ - echo "mariadb-server" mysql-server/root_password_again password '${DB_ROOT_PASSWORD}'; \ - } | debconf-set-selections; \ - apt-get update; \ - apt-get install -y \ - "mariadb-server" \ - socat \ - ; \ - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; fi - -RUN if [ "$DB_DISABLE_INCLUDED" = "false" ] ; then sed -ie "s/^bind-address\s*=\s*127\.0\.0\.1$/#bind-address = 0.0.0.0/" /etc/mysql/my.cnf; fi - -# Install FFmpeg - -RUN apt update --fix-missing -RUN apt install -y software-properties-common \ - libfreetype6-dev \ - libgnutls28-dev \ - libmp3lame-dev \ - libass-dev \ - libogg-dev \ - libtheora-dev \ - libvorbis-dev \ - libvpx-dev \ - libwebp-dev \ - libssh2-1-dev \ - libopus-dev \ - librtmp-dev \ - libx264-dev \ - libx265-dev \ - yasm -RUN apt install -y \ - build-essential \ - bzip2 \ - coreutils \ - procps \ - gnutls-bin \ - nasm \ - tar \ - x264 - -RUN apt install -y \ - ffmpeg \ - git \ - make \ - g++ \ - gcc \ - pkg-config \ - python3 \ - wget \ - tar \ - sudo \ - xz-utils - + DB_DISABLE_INCLUDED=$VAR_EXCLUDE_DB \ + NODE_MAJOR=18 \ + INSTALL_NODE=$VAR_INSTALL_NODE WORKDIR /home/Shinobi -COPY . . -#RUN rm -rf /home/Shinobi/plugins -COPY ./plugins /home/Shinobi/plugins -RUN chmod -R 777 /home/Shinobi/plugins -RUN npm i npm@latest -g && \ - npm install --unsafe-perm && \ - npm install pm2 -g -COPY ./Docker/pm2.yml ./ +COPY . ./ -# Copy default configuration files -# COPY ./config/conf.json ./config/super.json /home/Shinobi/ -RUN chmod -f +x /home/Shinobi/Docker/init.sh -RUN sed -i -e 's/\r//g' /home/Shinobi/Docker/init.sh -# RUN chmod -f +x /home/Shinobi/shinobi +RUN sh /home/Shinobi/Docker/install_dependencies.sh VOLUME ["/home/Shinobi/videos"] VOLUME ["/home/Shinobi/libs/customAutoLoad"] VOLUME ["/config"] -VOLUME ["/var/lib/mysql"] EXPOSE 8080 443 21 25 -ENTRYPOINT ["sh","/home/Shinobi/Docker/init.sh"] +ENTRYPOINT ["sh", "/home/Shinobi/Docker/init.sh"] CMD [ "pm2-docker", "/home/Shinobi/Docker/pm2.yml" ]