From d5d3fa6d29143a70b967076077b4fcf11a9e4757 Mon Sep 17 00:00:00 2001 From: Chris Veilleux Date: Fri, 17 Apr 2020 19:12:18 -0500 Subject: [PATCH 1/5] New Jenkinsfile for continuous integration --- Jenkinsfile | 136 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..279b92f --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,136 @@ +pipeline { + agent any + options { + // Running builds concurrently could cause a race condition with + // building the Docker image. + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '5')) + } + environment { + // Some branches have a "/" in their name (e.g. feature/new-and-cool) + // Some commands, such as those tha deal with directories, don't + // play nice with this naming convention. Define an alias for the + // branch name that can be used in these scenarios. + BRANCH_ALIAS = sh( + script: 'echo $BRANCH_NAME | sed -e "s#/#-#g"', + returnStdout: true + ).trim() + } + stages { + stage('Build, Format & Lint') { + // Build a Docker image containing the Precise application and all + // prerequisites. Use git to determine the list of files changed. + // Filter the list of changed files into a list of Python modules. + // Pass the list of Python files changed into the Black code + // formatter. Build will fail if Black finds any changes to make. + // If Black check passes, run PyLint against the same set of Python + // modules. Build will fail if lint is found in code. + when { + changeRequest target: 'dev' + } + steps { + sh 'docker build -f test/Dockerfile -t precise:${BRANCH_ALIAS} .' + sh 'git fetch origin dev' + sh 'git --no-pager diff --name-only FETCH_HEAD > $HOME/code-quality/change-set.txt' + sh 'docker run \ + -v $HOME/code-quality/:/root/code-quality \ + --entrypoint /bin/bash \ + precise:${BRANCH_ALIAS} \ + -x -c "grep -F .py /root/code-quality/change-set.txt | xargs black --check"' + sh 'docker run \ + -v $HOME/code-quality/:/root/code-quality \ + --entrypoint /bin/bash \ + precise:${BRANCH_ALIAS} \ + -x -c "grep -F .py /root/code-quality/change-set.txt | xargs pylint"' + } + } + stage('Run Tests') { + // Run the unit and/or integration tests defined within the repository + when { + anyOf { + branch 'dev' + branch 'master' + changeRequest target: 'dev' + } + } + steps { + echo 'Building Precise Testing Docker Image' + sh 'docker build -f test/Dockerfile -t precise:${BRANCH_ALIAS} .' + echo 'Precise Test Suite' + timeout(time: 5, unit: 'MINUTES') + { + sh 'docker run \ + -v "$HOME/allure/precise/:/root/allure" \ + precise:${BRANCH_ALIAS}' + } + } + } + } + post { + cleanup { + sh( + label: 'Docker Container and Image Cleanup', + script: ''' + docker container prune --force; + docker image prune --force; + ''' + ) + } + failure { + // Send failure email containing a link to the Jenkins build + // the results report and the console log messages to Mycroft + // developers, the developers of the pull request and the + // developers that caused the build to fail. + echo 'Sending Failure Email' + emailext ( + subject: "FAILURE - Precise Build - ${BRANCH_NAME} #${BUILD_NUMBER}", + body: """ +

+ Follow the link below to see details regarding + the cause of the failure. Once a fix is pushed, + this job will be re-run automatically. +

+
+

Jenkins Build Details

+
""", + replyTo: 'devops@mycroft.ai', + to: 'chris.veilleux@mycroft.ai', + recipientProviders: [ + [$class: 'RequesterRecipientProvider'], + [$class:'CulpritsRecipientProvider'], + [$class:'DevelopersRecipientProvider'] + ] + ) + } + success { + // Send success email containing a link to the Jenkins build + // and the results report to Mycroft developers, the developers + // of the pull request and the developers that caused the + // last failed build. + echo 'Sending Success Email' + emailext ( + subject: "SUCCESS - Precise Tests - Build ${BRANCH_NAME} #${BUILD_NUMBER}", + body: """ +

+ Build completed without issue. No further action required. + Build details can be found by following the link below. +

+
+

+ + Jenkins Build Details + +  (Requires account on Mycroft's Jenkins instance) +

+
""", + replyTo: 'devops@mycroft.ai', + to: 'chris.veilleux@mycroft.ai', + recipientProviders: [ + [$class: 'RequesterRecipientProvider'], + [$class:'CulpritsRecipientProvider'], + [$class:'DevelopersRecipientProvider'] + ] + ) + } + } +} From 3e1a24ea0312603e4301aa06ce089fe2d917a94f Mon Sep 17 00:00:00 2001 From: Chris Veilleux Date: Mon, 13 Apr 2020 14:01:34 -0500 Subject: [PATCH 2/5] remove ignore of .txt files --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index b0c1656..08a915c 100755 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ __pycache__/ *.net *.json *.pbtxt -*.txt *.wav # Data folders From cfc36aafb22536de714507cc5738ad07aa494b77 Mon Sep 17 00:00:00 2001 From: Chris Veilleux Date: Mon, 13 Apr 2020 14:01:13 -0500 Subject: [PATCH 3/5] new requirements files --- requirements/prod.txt | 14 ++++++++++++++ requirements/test.txt | 3 +++ 2 files changed, 17 insertions(+) create mode 100644 requirements/prod.txt create mode 100644 requirements/test.txt diff --git a/requirements/prod.txt b/requirements/prod.txt new file mode 100644 index 0000000..bf6b103 --- /dev/null +++ b/requirements/prod.txt @@ -0,0 +1,14 @@ +numpy==1.16 +tensorflow>=1.13,<1.14 # Must be on piwheels +sonopy +pyaudio +keras<=2.1.5 +h5py +wavio +typing +prettyparse>=1.1.0 +precise-runner +attrs +fitipy<1.0 +speechpy-fast +pyache diff --git a/requirements/test.txt b/requirements/test.txt new file mode 100644 index 0000000..8ce95ae --- /dev/null +++ b/requirements/test.txt @@ -0,0 +1,3 @@ +pylint +black +pytest From 803a283e09ef1819f03ae6516afa25eb6cb3d4cd Mon Sep 17 00:00:00 2001 From: Chris Veilleux Date: Fri, 17 Apr 2020 20:06:11 -0500 Subject: [PATCH 4/5] added pylintrc file --- pylintrc | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pylintrc diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..bf5aa54 --- /dev/null +++ b/pylintrc @@ -0,0 +1,2 @@ +[MESSAGES CONTROL] +disable=C0330 From 735c4f91094c37a4358331b68a74a6f1b4c741a6 Mon Sep 17 00:00:00 2001 From: Chris Veilleux Date: Mon, 27 Apr 2020 12:34:08 -0500 Subject: [PATCH 5/5] Move lint and code format checks to devops repo --- Jenkinsfile | 37 +++++++++++++++---------------------- test/Dockerfile | 34 ++++++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 279b92f..50fe3c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,33 +15,22 @@ pipeline { script: 'echo $BRANCH_NAME | sed -e "s#/#-#g"', returnStdout: true ).trim() + //spawns GITHUB_USR and GITHUB_PSW environment variables + GITHUB=credentials('38b2e4a6-167a-40b2-be6f-d69be42c8190') } stages { - stage('Build, Format & Lint') { - // Build a Docker image containing the Precise application and all - // prerequisites. Use git to determine the list of files changed. - // Filter the list of changed files into a list of Python modules. - // Pass the list of Python files changed into the Black code - // formatter. Build will fail if Black finds any changes to make. - // If Black check passes, run PyLint against the same set of Python - // modules. Build will fail if lint is found in code. + stage('Lint & Format') { + // Run PyLint and Black to check code quality. when { changeRequest target: 'dev' } steps { - sh 'docker build -f test/Dockerfile -t precise:${BRANCH_ALIAS} .' - sh 'git fetch origin dev' - sh 'git --no-pager diff --name-only FETCH_HEAD > $HOME/code-quality/change-set.txt' - sh 'docker run \ - -v $HOME/code-quality/:/root/code-quality \ - --entrypoint /bin/bash \ - precise:${BRANCH_ALIAS} \ - -x -c "grep -F .py /root/code-quality/change-set.txt | xargs black --check"' - sh 'docker run \ - -v $HOME/code-quality/:/root/code-quality \ - --entrypoint /bin/bash \ - precise:${BRANCH_ALIAS} \ - -x -c "grep -F .py /root/code-quality/change-set.txt | xargs pylint"' + sh 'docker build \ + --build-arg github_api_key=$GITHUB_PSW \ + --file test/Dockerfile \ + --target code-checker \ + -t precise:${BRANCH_ALIAS} .' + sh 'docker run precise:${BRANCH_ALIAS}' } } stage('Run Tests') { @@ -55,7 +44,11 @@ pipeline { } steps { echo 'Building Precise Testing Docker Image' - sh 'docker build -f test/Dockerfile -t precise:${BRANCH_ALIAS} .' + sh 'docker build \ + --build-arg github_api_key=$GITHUB_PSW \ + --file test/Dockerfile \ + --target test-runner \ + -t precise:${BRANCH_ALIAS} .' echo 'Precise Test Suite' timeout(time: 5, unit: 'MINUTES') { diff --git a/test/Dockerfile b/test/Dockerfile index fb3e000..5d9af2f 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,10 +1,32 @@ -FROM python:3.7-slim +# This dockerfile is for continuous integration of the mycroft-precise repostiory + +# Build an environment that can run the Precise wake word spotter. +FROM python:3.7-slim as precise-build ENV TERM linux ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get -y install git python3-scipy cython libhdf5-dev python3-h5py portaudio19-dev swig libpulse-dev libatlas-base-dev -ADD . mycroft-precise -WORKDIR mycroft-precise -RUN pip install . -RUN pip install pytest -ENV PYTHONPATH /mycroft-precise +RUN mkdir -p /root/allure /opt/mycroft/mycroft-precise /root/code-quality +WORKDIR /opt/mycroft +COPY requirements/test.txt mycroft-precise/requirements/ +RUN pip install -r mycroft-precise/requirements/test.txt +COPY requirements/prod.txt mycroft-precise/requirements/ +RUN pip install -r mycroft-precise/requirements/prod.txt +COPY . mycroft-precise + +# Clone the devops repository, which contiains helper scripts for some continuous +# integraion tasks. Run the code_check.py script which performs linting (using PyLint) +# and code formatting (using Black) +FROM precise-build as code-checker +ARG github_api_key +ENV GITHUB_API_KEY=$github_api_key +RUN pip install pipenv +RUN git clone https://$github_api_key@github.com/MycroftAI/devops.git +WORKDIR /opt/mycroft/devops/jenkins +RUN git checkout continuous_integration +RUN pipenv install +ENTRYPOINT ["pipenv", "run", "python","-m", "pipeline.code_check", "--repository", "mycroft-precise", "--pull-request", "PR-149"] + +# Run the tests defined in the precise repository +FROM precise-build as test-runner +WORKDIR /opt/mycroft/mycroft-precise ENTRYPOINT ["pytest"]