Add option to relock_dependencies.sh to produce requirement files from the existing Pipfile.lock

Otherwise, the default is to create a lock file, and then create the requirements files. The motivation of this change is that the validate_reqs_files job on CI wasn't deterministic, as it was failing everytime some dependency was updated (and therefore, the produced dependency file was updated and different to the existing in the codebase)

scripts/circle/compare_reqs.sh

@@ -2,10 +2,11 @@
 
 set -e
 
-# update lock and build requirements files
-yes | ./scripts/dependencies/relock_dependencies.sh circle-requirements
+# Update lock and build requirements files.
+# Use option -k to keep the Pipfile.lock file, so the process is deterministic with respect to that file.
+yes | ./scripts/dependencies/relock_dependencies.sh -k circle-requirements
 
-echo "---- validating requirements.txt ----"
+echo "---- Validating requirements.txt ----"
 REQSHASH=$(md5sum requirements.txt | cut -d ' ' -f1)
 TESTHASH=$(md5sum circle-requirements.txt | cut -d ' ' -f1)
 
@@ -13,7 +14,6 @@ echo "- $REQSHASH"
 echo "- $TESTHASH"
 if [ $REQSHASH == $TESTHASH ]; then
     echo "- requirements.txt is valid ...."
-
 else
     echo "- requirements.txt contains inconsistencies ...."
     echo "- you may want to run `pipenv sync --dev` and then ./scripts/dependencies/relock_dependencies.sh ...."
@@ -22,7 +22,7 @@ else
     exit 2
 fi
 
-echo "---- validating dev-requirements.txt ----"
+echo "---- Validating dev-requirements.txt ----"
 REQSHASH=$(md5sum dev-requirements.txt | cut -d ' ' -f1)
 TESTHASH=$(md5sum dev-circle-requirements.txt | cut -d ' ' -f1)
 
@@ -31,7 +31,6 @@ echo "- $TESTHASH"
 
 if [ $REQSHASH == $TESTHASH ]; then
     echo "- dev-requirements.txt is valid ...."
-
 else
     echo "- dev-requirements.txt contains inconsistencies ...."
     echo "- you may want to run `pipenv sync --dev` and then ./scripts/dependencies/relock_dependencies.sh ...."

scripts/dependencies/relock_dependencies.sh

@@ -1,17 +1,33 @@
 #!/usr/bin/env bash
 
+# Parse optional flag -k, to be used when we want to base the process on an existing Pipfile.lock
+KEEP_LOCK=false
+OPTIND=1
+while getopts 'k' opt; do
+    case $opt in
+        k) KEEP_LOCK=true ;;
+        *) echo 'Error in command line parsing' >&2
+           exit 1
+    esac
+done
+shift "$(( OPTIND - 1 ))"
+
 # can change output file names with relock_dependencies.sh <prefix>
 PREFIX=${1:-requirements}
 
 # these steps might fail, but that's okay.
-echo "Removing existing lock files..."
+if ! "$KEEP_LOCK"; then
+    echo "Removing existing Pipfile.lock file"
+    rm -f Pipfile.lock
+fi
+
+echo "Removing existing requirement files"
 pipenv --rm
-rm -f Pipfile.lock
 rm -f $PREFIX.txt
 rm -f dev-$PREFIX.txt
 rm -f docs-$PREFIX.txt
 
-echo "Removing pip and pipenv system cache..."
+echo "Removing pip and pipenv system cache"
 rm -r ~/.cache/pip ~/.cache/pipenv
 
 # start enforcing failures