diff --git a/.circleci/config.yml b/.circleci/config.yml index f154f8165..f950b2d4c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,6 +172,10 @@ workflows: only: - main jobs: + - validate_reqs_files: + filters: + tags: + only: /.*/ - pipenv_install_36: filters: tags: @@ -184,10 +188,6 @@ workflows: filters: tags: only: /.*/ - - validate_reqs_files: - filters: - tags: - only: /.*/ - pip_install_37: filters: tags: @@ -200,6 +200,14 @@ workflows: filters: tags: only: /.*/ + - pipenv_install_39: + filters: + tags: + only: /.*/ + - pip_install_39: + filters: + tags: + only: /.*/ - statistical_tests: filters: tags: @@ -333,6 +341,12 @@ python_38_base: &python_38_base docker: - image: circleci/python:3.8 +python_39_base: &python_39_base + parallelism: 1 + working_directory: ~/nucypher + docker: + - image: circleci/python:3.9 + commands: pipenv_install: @@ -474,6 +488,8 @@ jobs: <<: *python_36_base steps: - checkout + - run: sudo chown -R circleci:circleci /usr/local/bin + - run: sudo chown -R circleci:circleci /usr/local/lib/python3.6/site-packages - pip_install pipenv_install_36: @@ -503,6 +519,8 @@ jobs: <<: *python_38_base steps: - checkout + - run: sudo chown -R circleci:circleci /usr/local/bin + - run: sudo chown -R circleci:circleci /usr/local/lib/python3.8/site-packages - pip_install pipenv_install_38: @@ -511,6 +529,21 @@ jobs: - pipenv_install: python_version: "3.8" + # Python 3.9 + pip_install_39: + <<: *python_39_base + steps: + - checkout + - run: sudo chown -R circleci:circleci /usr/local/bin + - run: sudo chown -R circleci:circleci /usr/local/lib/python3.9/site-packages + - pip_install + + pipenv_install_39: + <<: *python_39_base + steps: + - pipenv_install: + python_version: "3.9" + integration: <<: *python_37_base parallelism: 2 @@ -662,6 +695,15 @@ jobs: - run: name: Run Requirements comparison command: ./scripts/circle/compare_reqs.sh + - run: + name: Prepare Requirement Files for Storage as Artifacts + command: | + mkdir -p /tmp/reqs + cp *requirements.txt /tmp/reqs + when: on_fail + - store_artifacts: + path: /tmp/reqs + destination: reqs estimate_gas: <<: *python_37_base diff --git a/newsfragments/2499.misc.rst b/newsfragments/2499.misc.rst new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/circle/compare_reqs.sh b/scripts/circle/compare_reqs.sh index 761b6514f..d02c73642 100755 --- a/scripts/circle/compare_reqs.sh +++ b/scripts/circle/compare_reqs.sh @@ -2,40 +2,39 @@ set -e -# update lock and build requirements files -yes | ./scripts/circle/rebuild_pipenv.sh circlereqs +# Update lock and build requirements files. +# Use option -k to keep the Pipfile.lock file, so the process is deterministic with respect to that file. +yes | ./scripts/dependencies/relock_dependencies.sh -k circle-requirements -echo "---- validating requirements.txt ----" +echo "---- Validating requirements.txt ----" REQSHASH=$(md5sum requirements.txt | cut -d ' ' -f1) -TESTHASH=$(md5sum circlereqs.txt | cut -d ' ' -f1) +TESTHASH=$(md5sum circle-requirements.txt | cut -d ' ' -f1) echo "- $REQSHASH" echo "- $TESTHASH" if [ $REQSHASH == $TESTHASH ]; then echo "- requirements.txt is valid ...." - else echo "- requirements.txt contains inconsistencies ...." - echo "- you may want to run `pipenv sync --dev` and then ./scripts/circle/rebuild_pipenv.sh ...." + echo "- you may want to run `pipenv sync --dev` and then ./scripts/dependencies/relock_dependencies.sh ...." echo "- which will rebuild your *requirements.txt files ...." - diff requirements.txt circlereqs.txt + diff requirements.txt circle-requirements.txt exit 2 fi -echo "---- validating dev-requirements.txt ----" +echo "---- Validating dev-requirements.txt ----" REQSHASH=$(md5sum dev-requirements.txt | cut -d ' ' -f1) -TESTHASH=$(md5sum dev-circlereqs.txt | cut -d ' ' -f1) +TESTHASH=$(md5sum dev-circle-requirements.txt | cut -d ' ' -f1) echo "- $REQSHASH" echo "- $TESTHASH" if [ $REQSHASH == $TESTHASH ]; then echo "- dev-requirements.txt is valid ...." - else echo "- dev-requirements.txt contains inconsistencies ...." - echo "- you may want to run `pipenv sync --dev` and then ./scripts/circle/rebuild_pipenv.sh ...." + echo "- you may want to run `pipenv sync --dev` and then ./scripts/dependencies/relock_dependencies.sh ...." echo "- which will rebuild your *requirements.txt files ...." - diff dev-requirements.txt dev-circlereqs.txt + diff dev-requirements.txt dev-circle-requirements.txt exit 2 fi diff --git a/scripts/circle/rebuild_pipenv.sh b/scripts/circle/rebuild_pipenv.sh deleted file mode 100755 index 784254549..000000000 --- a/scripts/circle/rebuild_pipenv.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# can change output file names with rebuild_pipenv.sh -PREFIX=${1:-requirements} - -read -p "Ok if we update your pip and setuptools? (type y or Y) " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]] -then - pip install --upgrade pip - pip install --upgrade setuptools -fi - -rm -f $PREFIX.txt -rm -f dev-$PREFIX.txt - -touch $PREFIX.txt -touch dev-$PREFIX.txt - -set -e -echo "rebuilding Pipfile.lock... this may take awhile." - -echo "bulding dev-$PREFIX.txt" -pipenv lock --clear --pre --requirements --dev > dev-$PREFIX.txt - -echo "building $PREFIX.txt" -pipenv lock --clear --pre --requirements > $PREFIX.txt diff --git a/scripts/dependencies/relock_dependencies.sh b/scripts/dependencies/relock_dependencies.sh index 6f38cc8ed..ef8d4351f 100755 --- a/scripts/dependencies/relock_dependencies.sh +++ b/scripts/dependencies/relock_dependencies.sh @@ -1,17 +1,33 @@ #!/usr/bin/env bash -# can change output file names with rebuild_pipenv.sh +# Parse optional flag -k, to be used when we want to base the process on an existing Pipfile.lock +KEEP_LOCK=false +OPTIND=1 +while getopts 'k' opt; do + case $opt in + k) KEEP_LOCK=true ;; + *) echo 'Error in command line parsing' >&2 + exit 1 + esac +done +shift "$(( OPTIND - 1 ))" + +# can change output file names with relock_dependencies.sh PREFIX=${1:-requirements} # these steps might fail, but that's okay. -echo "Removing existing lock files..." +if ! "$KEEP_LOCK"; then + echo "Removing existing Pipfile.lock file" + rm -f Pipfile.lock +fi + +echo "Removing existing requirement files" pipenv --rm -rm -f Pipfile.lock rm -f $PREFIX.txt rm -f dev-$PREFIX.txt rm -f docs-$PREFIX.txt -echo "Removing pip and pipenv system cache..." +echo "Removing pip and pipenv system cache" rm -r ~/.cache/pip ~/.cache/pipenv # start enforcing failures