Merge pull request #149 from MycroftAI/feature/continuous-integration

Add continuous integration
2020-04-29 08:56:36 +02:00 · 2020-04-29 08:56:36 +02:00 · ba191825a1
parent 04053a5839 735c4f9109
commit ba191825a1
6 changed files with 176 additions and 7 deletions
--- a/.gitignore
+++ b/.gitignore
@ -17,7 +17,6 @@ __pycache__/
 *.net
 *.json
 *.pbtxt
-*.txt
 *.wav

 # Data folders
--- a/129
+++ b/129
@ -0,0 +1,129 @@
+pipeline {
+    agent any
+    options {
+        // Running builds concurrently could cause a race condition with
+        // building the Docker image.
+        disableConcurrentBuilds()
+        buildDiscarder(logRotator(numToKeepStr: '5'))
+    }
+    environment {
+        // Some branches have a "/" in their name (e.g. feature/new-and-cool)
+        // Some commands, such as those tha deal with directories, don't
+        // play nice with this naming convention.  Define an alias for the
+        // branch name that can be used in these scenarios.
+        BRANCH_ALIAS = sh(
+            script: 'echo $BRANCH_NAME | sed -e "s#/#-#g"',
+            returnStdout: true
+        ).trim()
+        //spawns GITHUB_USR and GITHUB_PSW environment variables
+        GITHUB=credentials('38b2e4a6-167a-40b2-be6f-d69be42c8190')
+    }
+    stages {
+        stage('Lint & Format') {
+            // Run PyLint and Black to check code quality.
+            when {
+                changeRequest target: 'dev'
+            }
+            steps {
+                sh 'docker build \
+                    --build-arg github_api_key=$GITHUB_PSW \
+                    --file test/Dockerfile \
+                    --target code-checker \
+                    -t precise:${BRANCH_ALIAS} .'
+                sh 'docker run precise:${BRANCH_ALIAS}'
+            }
+        }
+        stage('Run Tests') {
+            // Run the unit and/or integration tests defined within the repository
+            when {
+                anyOf {
+                    branch 'dev'
+                    branch 'master'
+                    changeRequest target: 'dev'
+                }
+            }
+            steps {
+                echo 'Building Precise Testing Docker Image'
+                sh 'docker build \
+                    --build-arg github_api_key=$GITHUB_PSW \
+                    --file test/Dockerfile \
+                    --target test-runner \
+                    -t precise:${BRANCH_ALIAS} .'
+                echo 'Precise Test Suite'
+                timeout(time: 5, unit: 'MINUTES')
+                {
+                    sh 'docker run \
+                        -v "$HOME/allure/precise/:/root/allure" \
+                        precise:${BRANCH_ALIAS}'
+                }
+            }
+        }
+    }
+    post {
+        cleanup {
+            sh(
+                label: 'Docker Container and Image Cleanup',
+                script: '''
+                    docker container prune --force;
+                    docker image prune --force;
+                '''
+            )
+        }
+        failure {
+            // Send failure email containing a link to the Jenkins build
+            // the results report and the console log messages to Mycroft
+            // developers, the developers of the pull request and the
+            // developers that caused the build to fail.
+            echo 'Sending Failure Email'
+            emailext (
+                subject: "FAILURE - Precise Build - ${BRANCH_NAME} #${BUILD_NUMBER}",
+                body: """
+                    <p>
+                        Follow the link below to see details regarding
+                        the cause of the failure.  Once a fix is pushed,
+                        this job will be re-run automatically.
+                    </p>
+                    <br>
+                    <p><a href='${BUILD_URL}'>Jenkins Build Details</a></p>
+                    <br>""",
+                replyTo: 'devops@mycroft.ai',
+                to: 'chris.veilleux@mycroft.ai',
+                recipientProviders: [
+                    [$class: 'RequesterRecipientProvider'],
+                    [$class:'CulpritsRecipientProvider'],
+                    [$class:'DevelopersRecipientProvider']
+                ]
+            )
+        }
+        success {
+            // Send success email containing a link to the Jenkins build
+            // and the results report to Mycroft developers, the developers
+            // of the pull request and the developers that caused the
+            // last failed build.
+            echo 'Sending Success Email'
+            emailext (
+                subject: "SUCCESS - Precise Tests - Build ${BRANCH_NAME} #${BUILD_NUMBER}",
+                body: """
+                    <p>
+                        Build completed without issue. No further action required.
+                        Build details can be found by following the link below.
+                    </p>
+                    <br>
+                    <p>
+                        <a href='${BUILD_URL}'>
+                            Jenkins Build Details
+                        </a>
+                        &nbsp(Requires account on Mycroft's Jenkins instance)
+                    </p>
+                    <br>""",
+                replyTo: 'devops@mycroft.ai',
+                to: 'chris.veilleux@mycroft.ai',
+                recipientProviders: [
+                    [$class: 'RequesterRecipientProvider'],
+                    [$class:'CulpritsRecipientProvider'],
+                    [$class:'DevelopersRecipientProvider']
+                ]
+            )
+        }
+    }
+}
--- a/2
+++ b/2
@ -0,0 +1,2 @@
+[MESSAGES CONTROL]
+disable=C0330
--- a/requirements/prod.txt
+++ b/requirements/prod.txt
@ -0,0 +1,14 @@
+numpy==1.16
+tensorflow>=1.13,<1.14  # Must be on piwheels
+sonopy
+pyaudio
+keras<=2.1.5
+h5py
+wavio
+typing
+prettyparse>=1.1.0
+precise-runner
+attrs
+fitipy<1.0
+speechpy-fast
+pyache
--- a/requirements/test.txt
+++ b/requirements/test.txt
@ -0,0 +1,3 @@
+pylint
+black
+pytest
--- a/test/Dockerfile
+++ b/test/Dockerfile
@ -1,10 +1,32 @@
-FROM python:3.7-slim
+# This dockerfile is for continuous integration of the mycroft-precise repostiory
+
+# Build an environment that can run the Precise wake word spotter.
+FROM python:3.7-slim as precise-build
 ENV TERM linux
 ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update && apt-get -y install git python3-scipy cython libhdf5-dev python3-h5py portaudio19-dev swig libpulse-dev libatlas-base-dev
-ADD . mycroft-precise
-WORKDIR mycroft-precise
-RUN pip install .
-RUN pip install pytest
-ENV PYTHONPATH /mycroft-precise
+RUN mkdir -p /root/allure /opt/mycroft/mycroft-precise /root/code-quality
+WORKDIR /opt/mycroft
+COPY requirements/test.txt mycroft-precise/requirements/
+RUN pip install -r mycroft-precise/requirements/test.txt
+COPY requirements/prod.txt mycroft-precise/requirements/
+RUN pip install -r mycroft-precise/requirements/prod.txt
+COPY . mycroft-precise
+
+# Clone the devops repository, which contiains helper scripts for some continuous
+# integraion tasks. Run the code_check.py script which performs linting (using PyLint)
+# and code formatting (using Black)
+FROM precise-build as code-checker
+ARG github_api_key
+ENV GITHUB_API_KEY=$github_api_key
+RUN pip install pipenv
+RUN git clone https://$github_api_key@github.com/MycroftAI/devops.git
+WORKDIR /opt/mycroft/devops/jenkins
+RUN git checkout continuous_integration
+RUN pipenv install
+ENTRYPOINT ["pipenv", "run", "python","-m", "pipeline.code_check", "--repository", "mycroft-precise", "--pull-request", "PR-149"]
+
+# Run the tests defined in the precise repository
+FROM precise-build as test-runner
+WORKDIR /opt/mycroft/mycroft-precise
 ENTRYPOINT ["pytest"]