nucypher/.circleci/config.yml

754 lines
21 KiB
YAML

version: 2.1
orbs:
codecov: codecov/codecov@1.0.5
workflows:
version: 2
test_build_deploy:
jobs:
- pip_install_37:
filters:
tags:
only: /.*/
- contracts:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- basics:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- network:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- character:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- agents:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- actors:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- deployers:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- cli:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- tests_ok:
filters:
tags:
only: /.*/
requires:
- basics
- network
- character
- cli
- deployers
# - agents # These jobs are left out to not block execution of workflow
# - actors
# - contracts
- build_dev_docker_images:
filters:
tags:
only: /.*/
requires:
- basics
- finnegans_wake_demo:
filters:
tags:
only: /.*/
requires:
- build_dev_docker_images
- heartbeat_demo:
filters:
tags:
only: /.*/
requires:
- build_dev_docker_images
- estimate_gas:
filters:
tags:
only: /.*/
requires:
- tests_ok
- build_docs:
filters:
tags:
only: /.*/
requires:
- tests_ok
- test_build:
filters:
tags:
only: /.*/
requires:
- build_docs
- heartbeat_demo
- finnegans_wake_demo
- build_docker:
filters:
tags:
only: /v[0-9]+.*/
branches:
only: master
requires:
- test_build
- test_deploy:
context: "NuCypher PyPI"
requires:
- test_build
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
- publish_docker_experimental:
context: "NuCypher Docker"
requires:
- build_docker
filters:
tags:
only: /.*/
branches:
only: master
- request_publication_approval:
type: approval
requires:
- test_deploy
- build_docker
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
- publish_pypi:
context: "NuCypher PyPI"
requires:
- request_publication_approval
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
- publish_docker:
context: "NuCypher Docker"
requires:
- request_publication_approval
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
nightly:
triggers:
- schedule:
cron: "0 6 * * *" # Scheduled for 0600 UTC (0800 CEST, 2300 PDT)
filters:
branches:
only:
- master
jobs:
- pipenv_install_36:
filters:
tags:
only: /.*/
- pip_install_36:
filters:
tags:
only: /.*/
- pipenv_install_37:
filters:
tags:
only: /.*/
- validate_reqs_files:
filters:
tags:
only: /.*/
- pip_install_37:
filters:
tags:
only: /.*/
- pipenv_install_38:
filters:
tags:
only: /.*/
- pip_install_38:
filters:
tags:
only: /.*/
- statistical_tests:
filters:
tags:
only: /.*/
requires:
- pip_install_37
python_36_base: &python_36_base
parallelism: 1
working_directory: ~/nucypher
docker:
- image: circleci/python:3.6
python_37_base: &python_37_base
parallelism: 1
working_directory: ~/nucypher
docker:
- image: circleci/python:3.7
python_38_base: &python_38_base
parallelism: 1
working_directory: ~/nucypher
docker:
- image: circleci/python:3.8
commands:
pipenv_install:
description: "Install Python dependencies with Pipenv"
parameters:
python_version:
type: string
default: "3.7"
steps:
- checkout
- run:
name: Install Python Dependencies with Pipenv
command: |
sudo apt update -y && sudo apt install python3-dev -y
pipenv sync --python << parameters.python_version >> --dev
- run:
name: Install Solidity Compiler (Pipenv Entrypoint)
command: pipenv run install-solc
- run:
name: Check NuCypher Python API Entrypoint
command: pipenv run python3 -c "import nucypher; print(nucypher.__version__)"
check_nucypher_entrypoints:
description: "Ensure both the python API and CLI entrypoints are usable"
steps:
- run:
name: Check NuCypher Python API Entrypoint
command: python3 -c "import nucypher; print(nucypher.__version__)"
- run:
name: Check NuCypher CLI Entrypoint
command: |
export PATH=~/.local/bin:$PATH
source ~/.bashrc
nucypher --help
pip_install:
description: "Install NuCypher with Pip"
steps:
- run:
name: Install Python Dependencies with Pip
command: pip3 install --user -e . -r requirements.txt || pip3 install --user -e . -r requirements.txt
- check_nucypher_entrypoints # Ensure Standard Installation Entry-points Work
- run:
name: Install Python Development Dependencies with Pip
command: pip3 install --user -e . -r dev-requirements.txt
- run:
name: Install Solidity Compiler
command: sudo python3 ./scripts/installation/install_solc.py
- check_nucypher_entrypoints
save_dependency_cache:
description: "Cache python installation files"
steps:
- run: sudo chown -R circleci:circleci /usr/local/bin
- run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
- save_cache:
key: pip-v3-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
paths:
- "~/.local/bin"
- "~/.local/lib/python3.7/site-packages"
- save_cache:
key: solc-v1-{{ checksum "nucypher/blockchain/eth/sol/__conf__.py" }}
paths:
- "/usr/local/bin/solc"
restore_dependency_cache:
description: "Own and restore cached python installation files"
steps:
- run: sudo chown -R circleci:circleci /usr/local/bin
- run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
- restore_cache: # ensure this step occurs *before* installing dependencies
key: pip-v3-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- restore_cache:
key: solc-v1-{{ checksum "nucypher/blockchain/eth/sol/__conf__.py" }}
prepare_environment:
description: "Checkout application code and Attach the Workspace"
steps:
- checkout
- restore_dependency_cache
- run:
name: "Create directory for test reports"
command: mkdir reports
prepare_dev_docker:
description: "access pre-build docker image"
steps:
- setup_remote_docker
- attach_workspace:
at: ~/docker-dev
- run:
name: "load docker"
command: docker load < ~/docker-dev/dev-docker-build.tar
codecov:
description: "Upload Coverage Report To codecov.io"
steps:
- codecov/upload:
file: reports/coverage.xml
capture_test_results:
description: "Store and Upload test results; Follow-up step for tests"
steps:
- codecov
- store_test_results:
path: reports
- run:
name: Prepare test logs for storage as artifacts
command: |
mkdir -p ~/test-logs/
mv ~/.cache/nucypher/log/nucypher.log ~/test-logs/nucypher-container-$CIRCLE_NODE_INDEX.log
when: always
- store_artifacts:
path: ~/test-logs/
build_and_save_test_docker:
description: "Build dev docker image for running tests against docker"
steps:
- checkout
- setup_remote_docker
- run:
name: Build Docker Image
command: docker-compose -f ./scripts/circle/docker-compose.yml build nucypher-circle-dev
- run:
name: mkdir
command: mkdir ~/docker-dev
- run:
name: save Docker Image
command: docker save circle:nucypher -o ~/docker-dev/dev-docker-build.tar
- persist_to_workspace:
root: ~/docker-dev
paths:
- "*.tar"
jobs:
# Python 3.6
pip_install_36:
<<: *python_36_base
steps:
- checkout
- pip_install
pipenv_install_36:
<<: *python_36_base
steps:
- pipenv_install:
python_version: "3.6"
# Python 3.7
# Note: Use the chown command to grant CircleCI access to dependency locations.
pip_install_37:
<<: *python_37_base
steps:
- checkout
- restore_dependency_cache # Depends on above checkout
- pip_install
- save_dependency_cache
pipenv_install_37:
<<: *python_37_base
steps:
- pipenv_install:
python_version: "3.7"
# Python 3.8
pip_install_38:
<<: *python_38_base
steps:
- checkout
- pip_install
pipenv_install_38:
<<: *python_38_base
steps:
- pipenv_install:
python_version: "3.8"
agents:
<<: *python_37_base
parallelism: 4
steps:
- prepare_environment
- run:
name: Blockchain Agent Tests
command: |
pytest $(circleci tests glob "tests/blockchain/eth/entities/agents/**/test_*.py" | circleci tests split --split-by=timings)
- capture_test_results
actors:
<<: *python_37_base
parallelism: 2
steps:
- prepare_environment
- run:
name: Blockchain Actor Tests
command: |
pytest $(circleci tests glob "tests/blockchain/eth/entities/actors/**/test_*.py" | circleci tests split --split-by=timings)
- capture_test_results
deployers:
<<: *python_37_base
parallelism: 2
steps:
- prepare_environment
- run:
name: Contract Deployer Tests
command: |
pytest $(circleci tests glob "tests/blockchain/eth/entities/deployers/test_*.py" | circleci tests split --split-by=timings)
- capture_test_results
contracts:
<<: *python_37_base
parallelism: 5
steps:
- prepare_environment
- run:
name: Ethereum Contract Unit Tests
command: |
pytest $(circleci tests glob "tests/blockchain/eth/contracts/**/test_*.py" | circleci tests split --split-by=timings)
- capture_test_results
basics:
<<: *python_37_base
parallelism: 1
steps:
- prepare_environment
- run:
name: Tests for Blockhain interfaces, Crypto functions, Node Configuration and Keystore
command: |
pytest $(circleci tests glob "tests/config/**/test_*.py" "tests/crypto/**/test_*.py" "tests/keystore/**/test_*.py" "tests/blockchain/eth/interfaces/**/test_*.py" "tests/blockchain/eth/clients/**/test_*.py" | circleci tests split --split-by=timings)
- capture_test_results
network:
<<: *python_37_base
steps:
- prepare_environment
- run:
name: Network Tests
command: |
pytest $(circleci tests glob "tests/network/**/test_*.py" | circleci tests split --split-by=timings)
- capture_test_results
character:
<<: *python_37_base
parallelism: 4
steps:
- prepare_environment
- run:
name: Character Tests
command: |
pytest $(circleci tests glob "tests/characters/**/test_*.py" "tests/learning/**/test_*.py" | circleci tests split --split-by=timings)
- capture_test_results
cli:
<<: *python_37_base
parallelism: 6
steps:
- prepare_environment
- run:
name: Nucypher CLI Tests
command: |
pytest $(circleci tests glob "tests/cli/**/test_*.py" | circleci tests split --split-by=timings)
- capture_test_results
tests_ok:
<<: *python_37_base
steps:
- run:
name: Nucypher CLI Tests
command: echo "Test modules succeeded"
build_dev_docker_images:
<<: *python_37_base
steps:
- build_and_save_test_docker
heartbeat_demo:
<<: *python_37_base
steps:
- checkout
- prepare_dev_docker
- run:
name: Run demo Ursula fleet, Alicia and the Doctor
command: ./scripts/circle/run_heartbeat_demo_docker-circle.sh
- store_artifacts:
path: /tmp/ursulas-logs
finnegans_wake_demo:
working_directory: ~/nucypher
<<: *python_37_base
steps:
- checkout
- prepare_dev_docker
- run:
name: Run demo Ursula fleet, Finnegans wake Demo code
command: ./scripts/circle/run_finnegans_wake_demo_docker-circle.sh
- store_artifacts:
path: /tmp/ursulas-logs
validate_reqs_files:
working_directory: ~/nucypher
<<: *python_37_base
steps:
- checkout
- run:
name: Run Requirements comparison
command: ./scripts/circle/compare_reqs.sh
estimate_gas:
<<: *python_37_base
steps:
- prepare_environment
- run:
name: Estimate Gas
command: |
python tests/metrics/estimate_gas.py
- store_artifacts:
path: tests/metrics/results/
build_docs:
<<: *python_37_base
steps:
- prepare_environment
- run:
name: Install Documentation Build Dependencies
command: pip3 install --user sphinx recommonmark sphinx-rtd-theme aafigure
- run:
name: Build Sphinx Documentation
command: |
cd docs
make html
- store_artifacts:
path: docs/build
test_build:
<<: *python_37_base
steps:
- checkout
- pip_install
- run:
name: Install Twine
command: pip install --user twine
- run:
name: Build Python Distribution Wheel
command: |
python setup.py sdist
python setup.py bdist_wheel
build_docker:
working_directory: ~/nucypher
docker:
- image: docker:18.06.1-ce-git
steps:
- checkout
- setup_remote_docker
- restore_cache:
keys:
- v1-{{ .Branch }}
paths:
- ~/docker/nucypher.tar
- run:
name: Load Docker Image Layer Cache
command: |
set +o pipefail
docker load -i ~/docker/nucypher.tar | true
- run:
name: Build Docker Image
command: |
docker build -f deploy/docker/Dockerfile --cache-from=nucypher -t nucypher/nucypher:circle .
- run:
name: Save Docker Image Layer Cache
command: |
mkdir -p ~/docker
docker save -o ~/docker/nucypher.tar nucypher/nucypher:circle
- save_cache:
key: v1-{{ .Branch }}-{{ epoch }}
paths:
- ~/docker/nucypher.tar
test_deploy:
<<: *python_37_base
steps:
- checkout
- pip_install
- run:
name: Install Twine
command: pip3 install --user twine
- run:
name: Verify git tag == __version__ (Test Deploy)
command: python setup.py verify
- run:
name: Initialize .pypirc (Test Deploy)
command: |
echo -e "[distutils]" >> ~/.pypirc
echo -e "index-servers = " >> ~/.pypirc
echo -e " pypi" >> ~/.pypirc
echo -e " testpypi" >> ~/.pypirc
echo -e "" >> ~/.pypirc
echo -e "[testpypi]" >> ~/.pypirc
echo -e "repository: https://test.pypi.org/legacy/" >> ~/.pypirc
echo -e "username: $TEST_PYPI_USERNAME" >> ~/.pypirc
echo -e "password: $TEST_PYPI_PASSWORD" >> ~/.pypirc
- run:
name: Build Python Wheel (Test Deploy)
command: |
python3 setup.py sdist
python3 setup.py bdist_wheel
- run:
name: upload to TestPyPI
command: python3 -m twine upload --repository testpypi dist/* --verbose
- run:
name: pip install from TestPyPI
command: |
pip3 install --user --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple nucypher --verbose
python3 -c "import nucypher; print(nucypher.__version__)"
publish_pypi:
<<: *python_37_base
steps:
- checkout
- pip_install
- run:
name: Install Twine
command: pip3 install --user twine
- run:
name: Verify git tag == __version__
command: python3 setup.py verify
- run:
name: Initialize .pypirc
command: |
echo -e "[distutils]" >> ~/.pypirc
echo -e "index-servers = " >> ~/.pypirc
echo -e " pypi" >> ~/.pypirc
echo -e "" >> ~/.pypirc
echo -e "[pypi]" >> ~/.pypirc
echo -e "username = $PYPI_USERNAME" >> ~/.pypirc
echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc
- run:
name: Build Python Wheel
command: |
python3 setup.py sdist
python3 setup.py bdist_wheel
- deploy:
name: Upload to PyPI
command: python3 -m twine upload dist/* --verbose
publish_docker_experimental:
working_directory: ~/nucypher
docker:
- image: docker:18.06.1-ce-git
steps:
- checkout
- setup_remote_docker
- restore_cache:
keys:
- v1-{{ .Branch }}
paths:
- ~/docker/nucypher.tar
- run:
name: Load Docker Image Layer Cache
command: |
set +o pipefail
docker load -i ~/docker/nucypher.tar | true
- deploy:
name: Push Latest NuCypher Docker Image
command: |
echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin
docker tag nucypher/nucypher:circle nucypher/nucypher:experimental
docker push nucypher/nucypher:experimental
publish_docker:
working_directory: ~/nucypher
docker:
- image: docker:18.06.1-ce-git
steps:
- checkout
- setup_remote_docker
- restore_cache:
keys:
- v1-{{ .Branch }}
paths:
- ~/docker/nucypher.tar
- run:
name: Load Docker Image Layer Cache
command: |
set +o pipefail
docker load -i ~/docker/nucypher.tar | true
- deploy:
name: Push Tagged NuCypher Docker Images
command: |
echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin
docker tag nucypher/nucypher:circle nucypher/nucypher:$CIRCLE_TAG
docker tag nucypher/nucypher:circle nucypher/nucypher:latest
docker push nucypher/nucypher:$CIRCLE_TAG
docker push nucypher/nucypher:latest
statistical_tests:
<<: *python_37_base
parallelism: 1
steps:
- prepare_environment
- run:
name: Statistical tests (e.g., sampling)
command: |
pytest --run-nightly --no-cov tests/blockchain/eth/entities/agents/test_sampling_distribution.py
- store_test_results:
path: reports
- run:
name: Prepare test logs for storage as artifacts
command: |
mkdir -p ~/test-logs/
mv ~/.cache/nucypher/log/nucypher.log ~/test-logs/nucypher-container-$CIRCLE_NODE_INDEX.log
when: always
- store_artifacts:
path: ~/test-logs/