nucypher/.circleci/config.yml

version: 2.1
workflows:
  version: 2
  test_build_deploy:
    jobs:
      - pip_install_37:
          filters:
            tags:
              only: /.*/
      - integration:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - unit:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - contracts:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - interfaces:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - unit
            - integration
      - characters:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - unit
            - integration
      - agents:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - unit
            - integration
      - actors:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - unit
            - integration
      - deployers:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - unit
            - integration
      - cli:
          context: "NuCypher Tests"
          filters:
            tags:
              only: /.*/
          requires:
            - unit
            - integration
      - tests_ok:
          filters:
            tags:
              only: /.*/
          requires:
            - actors
            - agents
            - interfaces
            - characters
            - cli
            - deployers
      - build_dev_docker_images:
          filters:
            tags:
              only: /.*/
      - finnegans_wake_demo:
          filters:
            tags:
              only: /.*/
          requires:
            - build_dev_docker_images
      - heartbeat_demo:
          filters:
            tags:
              only: /.*/
          requires:
            - build_dev_docker_images
      - estimate_gas:
          filters:
            tags:
              only: /.*/
          requires:
            - tests_ok
      - build_docs:
          filters:
            tags:
              only: /.*/
      - test_build:
          filters:
            tags:
              only: /.*/
          requires:
            - tests_ok
      - build_docker:
          filters:
            tags:
              only: /v[0-9]+.*/
            branches:
              only: main
          requires:
            - test_build
      - publish_docker_experimental:
          context: "NuCypher Docker"
          requires:
            - build_docker
          filters:
            tags:
              only: /.*/
            branches:
              only: main
      - request_publication_approval:
          type: approval
          requires:
            - build_docker
          filters:
            tags:
              only: /v[0-9]+.*/
            branches:
              ignore: /.*/
      - publish_pypi:
          context: "NuCypher PyPI"
          requires:
            - request_publication_approval
          filters:
            tags:
              only: /v[0-9]+.*/
            branches:
              ignore: /.*/
      - publish_docker:
          context: "NuCypher Docker"
          requires:
            - request_publication_approval
          filters:
            tags:
              only: /v[0-9]+.*/
            branches:
              ignore: /.*/
  nightly:
    triggers:
      - schedule:
          cron: "0 6 * * *"   # Scheduled for 0600 UTC (0800 CEST, 2300 PDT)
          filters:
            branches:
              only:
                - main
    jobs:
      - pipenv_install_36:
          filters:
            tags:
              only: /.*/
      - pip_install_36:
          filters:
            tags:
              only: /.*/
      - pipenv_install_37:
          filters:
            tags:
              only: /.*/
      - validate_reqs_files:
          filters:
            tags:
              only: /.*/
      - pip_install_37:
          filters:
            tags:
              only: /.*/
      - pipenv_install_38:
          filters:
            tags:
              only: /.*/
      - pip_install_38:
          filters:
            tags:
              only: /.*/
      - statistical_tests:
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - contracts:
          context: "Nightly"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - interfaces:
          context: "Nightly"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - characters:
          context: "Nightly"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - agents:
          context: "Nightly"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - actors:
          context: "Nightly"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - deployers:
          context: "Nightly"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - cli:
          context: "Nightly"
          filters:
            tags:
              only: /.*/
          requires:
            - pip_install_37
      - tests_ok:
          filters:
            tags:
              only: /.*/
          requires:
            - actors
            - agents
            - interfaces
            - characters
            - cli
            - deployers
      - build_dev_docker_images:
          filters:
            tags:
              only: /.*/
          requires:
            - tests_ok
      - finnegans_wake_demo:
          filters:
            tags:
              only: /.*/
          requires:
            - build_dev_docker_images
      - heartbeat_demo:
          filters:
            tags:
              only: /.*/
          requires:
            - build_dev_docker_images
      - estimate_gas:
          filters:
            tags:
              only: /.*/
          requires:
            - tests_ok
      - build_docs:
          filters:
            tags:
              only: /.*/
          requires:
            - tests_ok
      - test_build:
          filters:
            tags:
              only: /.*/
          requires:
            - build_docs
            - heartbeat_demo
            - finnegans_wake_demo
      - build_docker:
          filters:
            tags:
              only: /v[0-9]+.*/
            branches:
              only: main
          requires:
            - test_build


python_36_base: &python_36_base
  parallelism: 1
  working_directory: ~/nucypher
  docker:
    - image: circleci/python:3.6

python_37_base: &python_37_base
  parallelism: 1
  working_directory: ~/nucypher
  docker:
    - image: circleci/python:3.7


python_38_base: &python_38_base
  parallelism: 1
  working_directory: ~/nucypher
  docker:
    - image: circleci/python:3.8

commands:

  pipenv_install:
    description: "Install Python dependencies with Pipenv"
    parameters:
      python_version:
        type: string
        default: "3.7"
    steps:
      - checkout
      - run:
          name: Install Python Dependencies with Pipenv
          command: |
            sudo apt update -y && sudo apt install python3-dev -y
            pipenv sync --python << parameters.python_version >> --dev
      - run:
          name: Install Solidity Compiler (Pipenv Entrypoint)
          command: pipenv run install-solc
      - run:
          name: Check NuCypher Python API Entrypoint
          command: pipenv run python3 -c "import nucypher; print(nucypher.__version__)"

  check_nucypher_entrypoints:
    description: "Ensure both the python API and CLI entrypoints are usable"
    steps:
      - run:
          name: Check NuCypher Python API Entrypoint
          command: python3 -c "import nucypher; print(nucypher.__version__)"
      - run:
          name: Check NuCypher CLI Entrypoint
          command: |
            export PATH=~/.local/bin:$PATH
            source ~/.bashrc
            nucypher --help

  pip_install:
    description: "Install NuCypher with Pip"
    steps:
      - run:
          name: Install Python Development Dependencies with Pip
          command: pip3 install --user -e .[dev]
      - run:
          name: Install Solidity Compiler
          command: python3 ./scripts/installation/install_solc.py
      - check_nucypher_entrypoints

  save_dependency_cache:
    description: "Cache python installation files"
    steps:
      - run: sudo chown -R circleci:circleci /usr/local/bin
      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
      - save_cache:
          key: pip-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
          paths:
            - "~/.local/bin"
            - "~/.local/lib/python3.7/site-packages"
      - save_cache:
          key: solc-v2-{{ checksum "nucypher/blockchain/eth/sol/__conf__.py" }}
          paths:
            - "~/.solcx/"

  restore_dependency_cache:
    description: "Own and restore cached python installation files"
    steps:
      - run: sudo chown -R circleci:circleci /usr/local/bin
      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
      - restore_cache:  # ensure this step occurs *before* installing dependencies
          key: pip-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
      - restore_cache:
          key: solc-v2-{{ checksum "nucypher/blockchain/eth/sol/__conf__.py" }}

  prepare_environment:
    description: "Checkout application code and Attach the Workspace"
    steps:
      - checkout
      - restore_dependency_cache
      - run:
          name: "Create directory for test reports"
          command: mkdir reports

  run_test_suite:
    description: "Runs a group of tests, specified by name and contained in a file"
    steps:
      - run:
          name: Running Test Suite
          command: |
            cat test-names.tmp | tr '\n' ' ' > test-names.txt
            pytest -c .circleci/pytest.ini `cat test-names.txt`

  prepare_dev_docker:
    description: "access pre-build docker image"
    steps:
      - setup_remote_docker
      - attach_workspace:
          at: ~/docker-dev
      - run:
          name: "load docker"
          command: docker load < ~/docker-dev/dev-docker-build.tar

  capture_test_results:
    description: "Store and Upload test results; Follow-up step for tests"
    steps:
      - store_test_results:
          path: reports
      - store_artifacts:
          path: ~/.cache/nucypher/log/nucypher.log
          destination: logs
      - store_artifacts:
          path: test-names.txt
          destination: tests

  build_and_save_test_docker:
    description: "Build dev docker image for running tests against docker"
    steps:
      - checkout
      - setup_remote_docker
      - run:
          name: Build Docker Image
          command: docker-compose -f ./scripts/circle/docker-compose.yml build nucypher-circle-dev
      - run:
          name: mkdir
          command: mkdir ~/docker-dev
      - run:
          name: save Docker Image
          command: docker save circle:nucypher -o ~/docker-dev/dev-docker-build.tar
      - persist_to_workspace:
          root: ~/docker-dev
          paths:
            - "*.tar"


jobs:

  # Python 3.6
  pip_install_36:
    <<: *python_36_base
    steps:
      - checkout
      - pip_install

  pipenv_install_36:
    <<: *python_36_base
    steps:
      - pipenv_install:
          python_version: "3.6"

  # Python 3.7
  # Note: Use the chown command to grant CircleCI access to dependency locations.
  pip_install_37:
    <<: *python_37_base
    steps:
      - checkout
      - restore_dependency_cache  # Depends on above checkout
      - pip_install
      - save_dependency_cache

  pipenv_install_37:
    <<: *python_37_base
    steps:
      - pipenv_install:
          python_version: "3.7"

  # Python 3.8
  pip_install_38:
    <<: *python_38_base
    steps:
      - checkout
      - pip_install

  pipenv_install_38:
    <<: *python_38_base
    steps:
      - pipenv_install:
          python_version: "3.8"

  integration:
    <<: *python_37_base
    parallelism: 2
    steps:
      - prepare_environment
      - run:
          name: Preparing Integration Test Suite
          command: |
            circleci tests glob "tests/integration/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  unit:
    <<: *python_37_base
    parallelism: 1
    steps:
      - prepare_environment
      - run:
          name: Preparing Unit Test Suite
          command: |
            circleci tests glob "tests/unit/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  agents:
    <<: *python_37_base
    parallelism: 4
    steps:
      - prepare_environment
      - run:
          name: Preparing Blockchain Agent Tests
          command: |
            circleci tests glob "tests/acceptance/blockchain/agents/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  actors:
    <<: *python_37_base
    parallelism: 2
    steps:
      - prepare_environment
      - run:
          name: Preparing Blockchain Actor Tests
          command: |
            circleci tests glob "tests/acceptance/blockchain/actors/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  deployers:
    <<: *python_37_base
    parallelism: 2
    steps:
      - prepare_environment
      - run:
          name: Preparing Contract Deployer Tests
          command: |
            circleci tests glob "tests/acceptance/blockchain/deployers/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  contracts:
    <<: *python_37_base
    parallelism: 4
    steps:
      - prepare_environment
      - run:
          name: Preparing Ethereum Contract Unit Tests
          command: |
            circleci tests glob "tests/contracts/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  interfaces:
    <<: *python_37_base
    parallelism: 1
    steps:
      - prepare_environment
      - run:
          name: Preparing Tests for Blockhain interfaces, Crypto functions, Node Configuration and Datastore
          command: |
            circleci tests glob "tests/acceptance/blockchain/interfaces" "tests/acceptance/blockchain/clients" | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  characters:
    <<: *python_37_base
    parallelism: 4
    steps:
      - prepare_environment
      - run:
          name: Preparing Character Tests
          command: |
            circleci tests glob "tests/acceptance/characters/**/test_*.py" "tests/learning/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  cli:
    <<: *python_37_base
    parallelism: 6
    steps:
      - prepare_environment
      - run:
          name: Preparing Nucypher CLI Tests
          command: |
            circleci tests glob "tests/acceptance/cli/**/test_*.py" | circleci tests split --split-by=timings | tee test-names.tmp
      - run_test_suite
      - capture_test_results

  tests_ok:
    <<: *python_37_base
    steps:
      - run:
          name: Nucypher CLI Tests
          command: echo "Test modules succeeded"

  build_dev_docker_images:
    <<: *python_37_base
    steps:
      - build_and_save_test_docker

  heartbeat_demo:
    <<: *python_37_base
    steps:
      - checkout
      - prepare_dev_docker
      - run:
          name: Run demo Ursula fleet, Alicia and the Doctor
          command: ./scripts/circle/run_heartbeat_demo_docker-circle.sh
      - store_artifacts:
          path: /tmp/ursulas-logs

  finnegans_wake_demo:
    working_directory: ~/nucypher
    <<: *python_37_base
    steps:
      - checkout
      - prepare_dev_docker
      - run:
          name: Run demo Ursula fleet, Finnegans wake Demo code
          command: ./scripts/circle/run_finnegans_wake_demo_docker-circle.sh
      - store_artifacts:
          path: /tmp/ursulas-logs

  validate_reqs_files:
    working_directory: ~/nucypher
    <<: *python_37_base
    steps:
      - checkout
      - run:
          name: Run Requirements comparison
          command: ./scripts/circle/compare_reqs.sh

  estimate_gas:
    <<: *python_37_base
    steps:
      - prepare_environment
      - run:
          name: Install Nucypher
          command: pip3 install --user -e .[benchmark]
      - run:
          name: Estimate Gas
          command: python tests/metrics/estimate_gas.py
      - store_artifacts:
          path: tests/metrics/results/

  build_docs:
    <<: *python_37_base
    steps:
      - prepare_environment
      - run:
          name: Install Documentation Build Dependencies
          command: pip3 install --user .[docs]
      - run:
          name: Build Sphinx Documentation
          command: make docs
      - store_artifacts:
          path: docs/build

  test_build:
    <<: *python_37_base
    steps:
      - checkout
      - run:
          name: Install Twine
          command: pip3 install --user twine
      - run:
          name: Build Python Distribution
          command: make dist

  build_docker:
    working_directory: ~/nucypher
    docker:
      - image: docker:18.06.1-ce-git
    steps:
      - checkout
      - setup_remote_docker
      - restore_cache:
          keys:
            - v1-{{ .Branch }}
          paths:
            - ~/docker/nucypher.tar
      - run:
          name: Load Docker Image Layer Cache
          command: |
            set +o pipefail
            docker load -i ~/docker/nucypher.tar | true
      - run:
          name: Build Docker Image
          command: |
            docker build -f deploy/docker/Dockerfile --cache-from=nucypher -t nucypher/nucypher:circle .
      - run:
          name: Save Docker Image Layer Cache
          command: |
            mkdir -p ~/docker
            docker save -o ~/docker/nucypher.tar nucypher/nucypher:circle
      - save_cache:
          key: v1-{{ .Branch }}-{{ epoch }}
          paths:
            - ~/docker/nucypher.tar

  publish_pypi:
    <<: *python_37_base
    steps:
      - checkout
      - run:
          name: Install Twine
          command: pip3 install --user twine
      - run:
          name: Verify git tag == __version__
          command: python3 setup.py verify
      - run:
          name: Initialize .pypirc
          command: |
            echo -e "[distutils]" >> ~/.pypirc
            echo -e "index-servers = " >> ~/.pypirc
            echo -e "    pypi" >> ~/.pypirc
            echo -e "" >> ~/.pypirc
            echo -e "[pypi]" >> ~/.pypirc
            echo -e "username = $PYPI_USERNAME" >> ~/.pypirc
            echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc
      - run:
          name: Build Python Wheel
          command: make dist
      - deploy:
          name: Upload to PyPI
          command: python3 -m twine upload dist/* --verbose

  publish_docker_experimental:
    working_directory: ~/nucypher
    docker:
      - image: docker:18.06.1-ce-git
    steps:
      - checkout
      - setup_remote_docker
      - restore_cache:
          keys:
            - v1-{{ .Branch }}
          paths:
            - ~/docker/nucypher.tar
      - run:
          name: Load Docker Image Layer Cache
          command: |
            set +o pipefail
            docker load -i ~/docker/nucypher.tar | true
      - deploy:
          name: Push Latest NuCypher Docker Image
          command: |
            echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin
            docker tag nucypher/nucypher:circle nucypher/nucypher:experimental
            docker push nucypher/nucypher:experimental

  publish_docker:
    working_directory: ~/nucypher
    docker:
      - image: docker:18.06.1-ce-git
    steps:
      - checkout
      - setup_remote_docker
      - restore_cache:
          keys:
            - v1-{{ .Branch }}
          paths:
            - ~/docker/nucypher.tar
      - run:
          name: Load Docker Image Layer Cache
          command: |
            set +o pipefail
            docker load -i ~/docker/nucypher.tar | true
      - deploy:
          name: Push Tagged NuCypher Docker Images
          command: |
            echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin
            docker tag nucypher/nucypher:circle nucypher/nucypher:$CIRCLE_TAG
            docker tag nucypher/nucypher:circle nucypher/nucypher:latest
            docker push nucypher/nucypher:$CIRCLE_TAG
            docker push nucypher/nucypher:latest

  statistical_tests:
      <<: *python_37_base
      parallelism: 1
      steps:
      - prepare_environment
      - run:
          name: Statistical tests (e.g., sampling)
          command: |
            pytest -c .circleci/pytest.ini --run-nightly --no-cov tests/acceptance/blockchain/agents/test_sampling_distribution.py
      - store_test_results:
          path: reports
      - store_artifacts:
          path: ~/.cache/nucypher/log/nucypher.log
          destination: logs