nucypher/.circleci/config.yml

899 lines
24 KiB
YAML

version: 2.1
workflows:
version: 2
test_build_deploy:
jobs:
- pip_install_37:
filters:
tags:
only: /.*/
- integration:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- unit:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- contracts:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- interfaces:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- unit
- integration
- characters:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- unit
- integration
- agents:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- unit
- integration
- actors:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- unit
- integration
- deployers:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- unit
- integration
- cli:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- unit
- integration
- utilities:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- unit
- integration
- tests_ok:
filters:
tags:
only: /.*/
requires:
- actors
- agents
- interfaces
- characters
- cli
- deployers
- utilities
- build_dev_docker_images:
filters:
tags:
only: /.*/
- finnegans_wake_demo:
filters:
tags:
only: /.*/
requires:
- build_dev_docker_images
- heartbeat_demo:
filters:
tags:
only: /.*/
requires:
- build_dev_docker_images
- estimate_gas:
filters:
tags:
only: /.*/
requires:
- tests_ok
- build_docs:
filters:
tags:
only: /.*/
- test_build:
filters:
tags:
only: /.*/
requires:
- tests_ok
- build_docker:
filters:
tags:
only: /v[0-9]+.*/
branches:
only: main
requires:
- test_build
- publish_docker_experimental:
context: "NuCypher Docker"
requires:
- build_docker
filters:
tags:
only: /.*/
branches:
only: main
- request_publication_approval:
type: approval
requires:
- build_docker
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
- publish_pypi:
context: "NuCypher PyPI"
requires:
- request_publication_approval
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
- publish_docker:
context: "NuCypher Docker"
requires:
- request_publication_approval
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
nightly:
triggers:
- schedule:
cron: "0 6 * * *" # Scheduled for 0600 UTC (0800 CEST, 2300 PDT)
filters:
branches:
only:
- main
jobs:
- validate_reqs_files:
filters:
tags:
only: /.*/
- pipenv_install_36:
filters:
tags:
only: /.*/
- pip_install_36:
filters:
tags:
only: /.*/
- pipenv_install_37:
filters:
tags:
only: /.*/
- pip_install_37:
filters:
tags:
only: /.*/
- pipenv_install_38:
filters:
tags:
only: /.*/
- pip_install_38:
filters:
tags:
only: /.*/
- pipenv_install_39:
filters:
tags:
only: /.*/
- pip_install_39:
filters:
tags:
only: /.*/
- statistical_tests:
filters:
tags:
only: /.*/
requires:
- pip_install_37
- contracts:
context: "Nightly"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- interfaces:
context: "Nightly"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- characters:
context: "Nightly"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- agents:
context: "Nightly"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- actors:
context: "Nightly"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- deployers:
context: "Nightly"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- cli:
context: "Nightly"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- utilities:
context: "Nightly"
filters:
tags:
only: /.*/
requires:
- pip_install_37
- tests_ok:
filters:
tags:
only: /.*/
requires:
- actors
- agents
- interfaces
- characters
- cli
- deployers
- utilities
- build_dev_docker_images:
filters:
tags:
only: /.*/
requires:
- tests_ok
- finnegans_wake_demo:
filters:
tags:
only: /.*/
requires:
- build_dev_docker_images
- heartbeat_demo:
filters:
tags:
only: /.*/
requires:
- build_dev_docker_images
- estimate_gas:
filters:
tags:
only: /.*/
requires:
- tests_ok
- build_docs:
filters:
tags:
only: /.*/
requires:
- tests_ok
- test_build:
filters:
tags:
only: /.*/
requires:
- build_docs
- heartbeat_demo
- finnegans_wake_demo
- build_docker:
filters:
tags:
only: /v[0-9]+.*/
branches:
only: main
requires:
- test_build
python_36_base: &python_36_base
parallelism: 1
working_directory: ~/nucypher
docker:
- image: circleci/python:3.6
python_37_base: &python_37_base
parallelism: 1
working_directory: ~/nucypher
docker:
- image: circleci/python:3.7
python_38_base: &python_38_base
parallelism: 1
working_directory: ~/nucypher
docker:
- image: circleci/python:3.8
python_39_base: &python_39_base
parallelism: 1
working_directory: ~/nucypher
docker:
- image: circleci/python:3.9
commands:
pipenv_install:
description: "Install Python dependencies with Pipenv"
parameters:
python_version:
type: string
default: "3.7"
steps:
- checkout
- run:
name: Install Python Dependencies with Pipenv
command: |
sudo apt update -y && sudo apt install python3-dev -y
pipenv sync --python << parameters.python_version >> --dev
- run:
name: Install Solidity Compiler (Pipenv Entrypoint)
command: pipenv run install-solc
- run:
name: Check NuCypher Python API Entrypoint
command: pipenv run python3 -c "import nucypher; print(nucypher.__version__)"
check_nucypher_entrypoints:
description: "Ensure both the python API and CLI entrypoints are usable"
steps:
- run:
name: Check NuCypher Python API Entrypoint
command: python3 -c "import nucypher; print(nucypher.__version__)"
- run:
name: Check NuCypher CLI Entrypoint
command: |
export PATH=~/.local/bin:$PATH
source ~/.bashrc
nucypher --help
pip_install:
description: "Install NuCypher with Pip"
steps:
- run:
name: Install Python Development Dependencies with Pip
command: pip3 install --user -e .[dev]
- run:
name: Check that pip installed package is importable
command: python -c "import nucypher"
- run:
name: Install Solidity Compiler
command: python3 ./scripts/installation/install_solc.py
- check_nucypher_entrypoints
save_dependency_cache:
description: "Cache python installation files"
steps:
- run: sudo chown -R circleci:circleci /usr/local/bin
- run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
- save_cache:
key: pip-v6-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
paths:
- "~/.local/bin"
- "~/.local/lib/python3.7/site-packages"
- save_cache:
key: solc-v2-{{ checksum "nucypher/blockchain/eth/sol/__conf__.py" }}
paths:
- "~/.solcx/"
restore_dependency_cache:
description: "Own and restore cached python installation files"
steps:
- run: sudo chown -R circleci:circleci /usr/local/bin
- run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
- restore_cache: # ensure this step occurs *before* installing dependencies
key: pip-v6-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- restore_cache:
key: solc-v2-{{ checksum "nucypher/blockchain/eth/sol/__conf__.py" }}
prepare_environment:
description: "Checkout application code and Attach the Workspace"
steps:
- checkout
- restore_dependency_cache
- run:
name: "Create directory for test reports"
command: mkdir reports
run_test_suite:
description: "Runs a group of tests, specified by name and contained in a file"
steps:
- run:
name: Running Test Suite
command: |
cat test-names.tmp | tr '\n' ' ' > test-names.txt
pytest -c .circleci/pytest.ini `cat test-names.txt`
prepare_dev_docker:
description: "access pre-build docker image"
steps:
- setup_remote_docker
- attach_workspace:
at: ~/docker-dev
- run:
name: "load docker"
command: docker load < ~/docker-dev/dev-docker-build.tar
capture_test_results:
description: "Store and Upload test results; Follow-up step for tests"
steps:
- store_test_results:
path: reports
- store_artifacts:
path: ~/.cache/nucypher/log/nucypher.log
destination: logs
- store_artifacts:
path: test-names.txt
destination: tests
build_and_save_test_docker:
description: "Build dev docker image for running tests against docker"
steps:
- checkout
- setup_remote_docker
- run:
name: Build Docker Image
command: docker-compose -f ./scripts/circle/docker-compose.yml build nucypher-circle-dev
- run:
name: mkdir
command: mkdir ~/docker-dev
- run:
name: save Docker Image
command: docker save circle:nucypher -o ~/docker-dev/dev-docker-build.tar
- persist_to_workspace:
root: ~/docker-dev
paths:
- "*.tar"
jobs:
# Python 3.6
pip_install_36:
<<: *python_36_base
steps:
- checkout
- run: sudo chown -R circleci:circleci /usr/local/bin
- run: sudo chown -R circleci:circleci /usr/local/lib/python3.6/site-packages
- pip_install
pipenv_install_36:
<<: *python_36_base
steps:
- pipenv_install:
python_version: "3.6"
# Python 3.7
# Note: Use the chown command to grant CircleCI access to dependency locations.
pip_install_37:
<<: *python_37_base
steps:
- checkout
- restore_dependency_cache # Depends on above checkout
- pip_install
- save_dependency_cache
pipenv_install_37:
<<: *python_37_base
steps:
- pipenv_install:
python_version: "3.7"
# Python 3.8
pip_install_38:
<<: *python_38_base
steps:
- checkout
- run: sudo chown -R circleci:circleci /usr/local/bin
- run: sudo chown -R circleci:circleci /usr/local/lib/python3.8/site-packages
- pip_install
pipenv_install_38:
<<: *python_38_base
steps:
- pipenv_install:
python_version: "3.8"
# Python 3.9
pip_install_39:
<<: *python_39_base
steps:
- checkout
- run: sudo chown -R circleci:circleci /usr/local/bin
- run: sudo chown -R circleci:circleci /usr/local/lib/python3.9/site-packages
- pip_install
pipenv_install_39:
<<: *python_39_base
steps:
- pipenv_install:
python_version: "3.9"
integration:
<<: *python_37_base
parallelism: 2
steps:
- prepare_environment
- run:
name: Preparing Integration Test Suite
command: |
circleci tests glob "tests/integration/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
unit:
<<: *python_37_base
parallelism: 1
steps:
- prepare_environment
- run:
name: Preparing Unit Test Suite
command: |
circleci tests glob "tests/unit/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
agents:
<<: *python_37_base
parallelism: 4
steps:
- prepare_environment
- run:
name: Preparing Blockchain Agent Tests
command: |
circleci tests glob "tests/acceptance/blockchain/agents/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
actors:
<<: *python_37_base
parallelism: 2
steps:
- prepare_environment
- run:
name: Preparing Blockchain Actor Tests
command: |
circleci tests glob "tests/acceptance/blockchain/actors/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
deployers:
<<: *python_37_base
parallelism: 2
steps:
- prepare_environment
- run:
name: Preparing Contract Deployer Tests
command: |
circleci tests glob "tests/acceptance/blockchain/deployers/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
contracts:
<<: *python_37_base
parallelism: 4
steps:
- prepare_environment
- run:
name: Preparing Ethereum Contract Unit Tests
command: |
circleci tests glob "tests/contracts/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
interfaces:
<<: *python_37_base
parallelism: 1
steps:
- prepare_environment
- run:
name: Preparing Tests for Blockchain interfaces and Ethereum clients
command: |
circleci tests glob "tests/acceptance/blockchain/interfaces/**/test_*.py" "tests/acceptance/blockchain/clients/**/test_*.py" | tee test-names.tmp
- run_test_suite
- capture_test_results
characters:
<<: *python_37_base
parallelism: 4
steps:
- prepare_environment
- run:
name: Preparing Character, Discovery Loop (aka "Learning") and Network Tests
command: |
circleci tests glob "tests/acceptance/characters/**/test_*.py" "tests/acceptance/learning/**/test_*.py" "tests/acceptance/network/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
cli:
<<: *python_37_base
parallelism: 6
steps:
- prepare_environment
- run:
name: Preparing Nucypher CLI Tests
command: |
circleci tests glob "tests/acceptance/cli/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
utilities:
<<: *python_37_base
parallelism: 1
steps:
- prepare_environment
- run:
name: Preparing Nucypher Utilities Tests
command: |
circleci tests glob "tests/acceptance/utilities/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
- run_test_suite
- capture_test_results
tests_ok:
<<: *python_37_base
steps:
- run:
name: Nucypher CLI Tests
command: echo "Test modules succeeded"
build_dev_docker_images:
<<: *python_37_base
steps:
- build_and_save_test_docker
heartbeat_demo:
<<: *python_37_base
steps:
- checkout
- prepare_dev_docker
- run:
name: Run demo Ursula fleet, Alicia and the Doctor
command: ./scripts/circle/run_heartbeat_demo_docker-circle.sh
- store_artifacts:
path: /tmp/ursulas-logs
finnegans_wake_demo:
working_directory: ~/nucypher
<<: *python_37_base
steps:
- checkout
- prepare_dev_docker
- run:
name: Run demo Ursula fleet, Finnegans wake Demo code
command: ./scripts/circle/run_finnegans_wake_demo_docker-circle.sh
- store_artifacts:
path: /tmp/ursulas-logs
validate_reqs_files:
working_directory: ~/nucypher
<<: *python_37_base
steps:
- checkout
- run:
name: Run Requirements comparison
command: ./scripts/circle/compare_reqs.sh
- run:
name: Prepare Requirement Files for Storage as Artifacts
command: |
mkdir -p /tmp/reqs
cp *requirements.txt /tmp/reqs
when: on_fail
- store_artifacts:
path: /tmp/reqs
destination: reqs
estimate_gas:
<<: *python_37_base
steps:
- prepare_environment
- run:
name: Install Nucypher
command: pip3 install --user -e .[benchmark]
- run:
name: Estimate Gas
command: python tests/metrics/estimate_gas.py
- store_artifacts:
path: tests/metrics/results/
build_docs:
<<: *python_37_base
steps:
- prepare_environment
- run:
name: Install Documentation Build Dependencies (Workaround for problem with pip resolver. See 2493)
command: |
python -m pip install pip==20.2.4
pip3 --version
pip3 install --user -r docs-requirements.txt .
- run:
name: Build Sphinx Documentation
command: make docs
- store_artifacts:
path: docs/build
test_build:
<<: *python_37_base
steps:
- checkout
- run:
name: Install Twine
command: pip3 install --user twine
- run:
name: Build Python Distribution
command: make dist
build_docker:
working_directory: ~/nucypher
docker:
- image: cimg/python:3.8.6
steps:
- checkout
- setup_remote_docker
- restore_cache:
keys:
- v2-{{ .Branch }}-{{ arch }}
paths:
- ~/docker/nucypher.tar
- run:
name: Load Docker Image Layer Cache
command: |
set +o pipefail
docker load -i ~/docker/nucypher.tar | true
- run:
name: Build Docker Image
command: |
docker build -f deploy/docker/Dockerfile --cache-from=nucypher -t nucypher/nucypher:circle .
- run:
name: Save Docker Image Layer Cache
command: |
mkdir -p ~/docker
docker save -o ~/docker/nucypher.tar nucypher/nucypher:circle
- save_cache:
key: v2-{{ .Branch }}-{{ arch }}-{{ epoch }}
paths:
- ~/docker/nucypher.tar
publish_pypi:
<<: *python_37_base
steps:
- checkout
- run:
name: Install Twine
command: pip3 install --user twine
- run:
name: Verify git tag == __version__
command: python3 setup.py verify
- run:
name: Initialize .pypirc
command: |
echo -e "[distutils]" >> ~/.pypirc
echo -e "index-servers = " >> ~/.pypirc
echo -e " pypi" >> ~/.pypirc
echo -e "" >> ~/.pypirc
echo -e "[pypi]" >> ~/.pypirc
echo -e "username = $PYPI_USERNAME" >> ~/.pypirc
echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc
- run:
name: Build Python Wheel
command: make dist
- deploy:
name: Upload to PyPI
command: python3 -m twine upload dist/* --verbose
publish_docker_experimental:
working_directory: ~/nucypher
docker:
- image: cimg/python:3.8.6
steps:
- checkout
- setup_remote_docker
- restore_cache:
keys:
- v2-{{ .Branch }}-{{ arch }}
paths:
- ~/docker/nucypher.tar
- run:
name: Load Docker Image Layer Cache
command: |
set +o pipefail
docker load -i ~/docker/nucypher.tar | true
- deploy:
name: Push Latest NuCypher Docker Image
command: |
echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin
docker tag nucypher/nucypher:circle nucypher/nucypher:experimental
docker push nucypher/nucypher:experimental
publish_docker:
working_directory: ~/nucypher
docker:
- image: cimg/python:3.8.6
steps:
- checkout
- setup_remote_docker
- restore_cache:
keys:
- v2-{{ .Branch }}-{{ arch }}
paths:
- ~/docker/nucypher.tar
- run:
name: Load Docker Image Layer Cache
command: |
set +o pipefail
docker load -i ~/docker/nucypher.tar | true
- deploy:
name: Push Tagged NuCypher Docker Images
command: |
echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin
docker tag nucypher/nucypher:circle nucypher/nucypher:$CIRCLE_TAG
docker tag nucypher/nucypher:circle nucypher/nucypher:latest
docker push nucypher/nucypher:$CIRCLE_TAG
docker push nucypher/nucypher:latest
statistical_tests:
<<: *python_37_base
parallelism: 1
steps:
- prepare_environment
- run:
name: Statistical tests (e.g., sampling)
command: |
pytest -c .circleci/pytest.ini --run-nightly --no-cov tests/acceptance/blockchain/agents/test_sampling_distribution.py
- store_test_results:
path: reports
- store_artifacts:
path: ~/.cache/nucypher/log/nucypher.log
destination: logs