Merge pull request #2499 from cygnusv/fix-nightly

Fix nightly build

.circleci/config.yml

@@ -172,6 +172,10 @@ workflows:
               only:
                 - main
     jobs:
+      - validate_reqs_files:
+          filters:
+            tags:
+              only: /.*/
       - pipenv_install_36:
           filters:
             tags:
@@ -184,10 +188,6 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - validate_reqs_files:
-          filters:
-            tags:
-              only: /.*/
       - pip_install_37:
           filters:
             tags:
@@ -200,6 +200,14 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - pipenv_install_39:
+          filters:
+            tags:
+              only: /.*/
+      - pip_install_39:
+          filters:
+            tags:
+              only: /.*/
       - statistical_tests:
           filters:
             tags:
@@ -333,6 +341,12 @@ python_38_base: &python_38_base
   docker:
     - image: circleci/python:3.8
 
+python_39_base: &python_39_base
+  parallelism: 1
+  working_directory: ~/nucypher
+  docker:
+    - image: circleci/python:3.9
+
 commands:
 
   pipenv_install:
@@ -474,6 +488,8 @@ jobs:
     <<: *python_36_base
     steps:
       - checkout
+      - run: sudo chown -R circleci:circleci /usr/local/bin
+      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.6/site-packages
       - pip_install
 
   pipenv_install_36:
@@ -503,6 +519,8 @@ jobs:
     <<: *python_38_base
     steps:
       - checkout
+      - run: sudo chown -R circleci:circleci /usr/local/bin
+      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.8/site-packages
       - pip_install
 
   pipenv_install_38:
@@ -511,6 +529,21 @@ jobs:
       - pipenv_install:
           python_version: "3.8"
 
+  # Python 3.9
+  pip_install_39:
+    <<: *python_39_base
+    steps:
+    - checkout
+    - run: sudo chown -R circleci:circleci /usr/local/bin
+    - run: sudo chown -R circleci:circleci /usr/local/lib/python3.9/site-packages
+    - pip_install
+
+  pipenv_install_39:
+    <<: *python_39_base
+    steps:
+    - pipenv_install:
+        python_version: "3.9"
+
   integration:
     <<: *python_37_base
     parallelism: 2
@@ -662,6 +695,15 @@ jobs:
       - run:
           name: Run Requirements comparison
           command: ./scripts/circle/compare_reqs.sh
+      - run:
+          name: Prepare Requirement Files for Storage as Artifacts
+          command: |
+            mkdir -p /tmp/reqs
+            cp *requirements.txt /tmp/reqs
+          when: on_fail
+      - store_artifacts:
+          path: /tmp/reqs
+          destination: reqs
 
   estimate_gas:
     <<: *python_37_base

scripts/circle/compare_reqs.sh

@@ -2,40 +2,39 @@
 
 set -e
 
-# update lock and build requirements files
-yes | ./scripts/circle/rebuild_pipenv.sh circlereqs
+# Update lock and build requirements files.
+# Use option -k to keep the Pipfile.lock file, so the process is deterministic with respect to that file.
+yes | ./scripts/dependencies/relock_dependencies.sh -k circle-requirements
 
-echo "---- validating requirements.txt ----"
+echo "---- Validating requirements.txt ----"
 REQSHASH=$(md5sum requirements.txt | cut -d ' ' -f1)
-TESTHASH=$(md5sum circlereqs.txt | cut -d ' ' -f1)
+TESTHASH=$(md5sum circle-requirements.txt | cut -d ' ' -f1)
 
 echo "- $REQSHASH"
 echo "- $TESTHASH"
 if [ $REQSHASH == $TESTHASH ]; then
     echo "- requirements.txt is valid ...."
-
 else
     echo "- requirements.txt contains inconsistencies ...."
-    echo "- you may want to run `pipenv sync --dev` and then ./scripts/circle/rebuild_pipenv.sh ...."
+    echo "- you may want to run `pipenv sync --dev` and then ./scripts/dependencies/relock_dependencies.sh ...."
     echo "- which will rebuild your *requirements.txt files ...."
-    diff requirements.txt circlereqs.txt
+    diff requirements.txt circle-requirements.txt
     exit 2
 fi
 
-echo "---- validating dev-requirements.txt ----"
+echo "---- Validating dev-requirements.txt ----"
 REQSHASH=$(md5sum dev-requirements.txt | cut -d ' ' -f1)
-TESTHASH=$(md5sum dev-circlereqs.txt | cut -d ' ' -f1)
+TESTHASH=$(md5sum dev-circle-requirements.txt | cut -d ' ' -f1)
 
 echo "- $REQSHASH"
 echo "- $TESTHASH"
 
 if [ $REQSHASH == $TESTHASH ]; then
     echo "- dev-requirements.txt is valid ...."
-
 else
     echo "- dev-requirements.txt contains inconsistencies ...."
-    echo "- you may want to run `pipenv sync --dev` and then ./scripts/circle/rebuild_pipenv.sh ...."
+    echo "- you may want to run `pipenv sync --dev` and then ./scripts/dependencies/relock_dependencies.sh ...."
     echo "- which will rebuild your *requirements.txt files ...."
-    diff dev-requirements.txt dev-circlereqs.txt
+    diff dev-requirements.txt dev-circle-requirements.txt
     exit 2
 fi

scripts/circle/rebuild_pipenv.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-# can change output file names with rebuild_pipenv.sh <prefix>
-PREFIX=${1:-requirements}
-
-read -p "Ok if we update your pip and setuptools? (type y or Y) " -n 1 -r
-echo
-if [[ $REPLY =~ ^[Yy]$ ]]
-then
-    pip install --upgrade pip
-    pip install --upgrade setuptools
-fi
-
-rm -f $PREFIX.txt
-rm -f dev-$PREFIX.txt
-
-touch $PREFIX.txt
-touch dev-$PREFIX.txt
-
-set -e
-echo "rebuilding Pipfile.lock... this may take awhile."
-
-echo "bulding dev-$PREFIX.txt"
-pipenv lock --clear --pre --requirements --dev > dev-$PREFIX.txt
-
-echo "building $PREFIX.txt"
-pipenv lock --clear --pre --requirements > $PREFIX.txt

scripts/dependencies/relock_dependencies.sh

@@ -1,17 +1,33 @@
 #!/usr/bin/env bash
 
-# can change output file names with rebuild_pipenv.sh <prefix>
+# Parse optional flag -k, to be used when we want to base the process on an existing Pipfile.lock
+KEEP_LOCK=false
+OPTIND=1
+while getopts 'k' opt; do
+    case $opt in
+        k) KEEP_LOCK=true ;;
+        *) echo 'Error in command line parsing' >&2
+           exit 1
+    esac
+done
+shift "$(( OPTIND - 1 ))"
+
+# can change output file names with relock_dependencies.sh <prefix>
 PREFIX=${1:-requirements}
 
 # these steps might fail, but that's okay.
-echo "Removing existing lock files..."
+if ! "$KEEP_LOCK"; then
+    echo "Removing existing Pipfile.lock file"
+    rm -f Pipfile.lock
+fi
+
+echo "Removing existing requirement files"
 pipenv --rm
-rm -f Pipfile.lock
 rm -f $PREFIX.txt
 rm -f dev-$PREFIX.txt
 rm -f docs-$PREFIX.txt
 
-echo "Removing pip and pipenv system cache..."
+echo "Removing pip and pipenv system cache"
 rm -r ~/.cache/pip ~/.cache/pipenv
 
 # start enforcing failures