Merge pull request #2835 from KPrasch/docker

rust-python execution environment for CI
2022-01-07 12:45:56 -08:00 · 2022-01-07 12:45:56 -08:00 · f420caeb1c
parent af58f703d9 5ad41dad13
commit f420caeb1c
5 changed files with 123 additions and 36 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -386,26 +386,29 @@ python_36_base: &python_36_base
  parallelism: 1
  working_directory: ~/nucypher
  docker:
-    - image: circleci/python:3.6
+    - image: kprasch/rust-python:3.6.9
+      user: circleci

 python_37_base: &python_37_base
  parallelism: 1
  working_directory: ~/nucypher
  docker:
-    - image: circleci/python:3.7
-
+    - image: kprasch/rust-python:3.7.9
+      user: circleci

 python_38_base: &python_38_base
  parallelism: 1
  working_directory: ~/nucypher
  docker:
-    - image: circleci/python:3.8
+    - image: kprasch/rust-python:3.8.9
+      user: circleci

 python_39_base: &python_39_base
  parallelism: 1
  working_directory: ~/nucypher
  docker:
-    - image: circleci/python:3.9
+    - image: kprasch/rust-python:3.9.9
+      user: circleci

 commands:

@ -442,15 +445,34 @@ commands:
            source ~/.bashrc
            nucypher --help

+  chown_system_paths:
+    description: "Set ownership of system binaries and site-packages"
+    steps:
+      - run:
+          name: Chown system site-packages
+          command: sudo chown circleci:circleci -R /usr/local/lib/python3.7/site-packages/
+      - run:
+          name: Chown system binaries
+          command: sudo chown circleci:circleci -R /usr/local/bin
+
+  chown_user_paths:
+    steps:
+      - run:
+          name: Set user-local permissions of site-packages and binaries
+          command: |
+            sudo chown -R circleci:circleci ~/.local/bin
+            sudo chown -R circleci:circleci ~/.local/lib/python3.7/site-packages
+
  pip_install:
    description: "Install NuCypher with Pip"
    steps:
+      - chown_system_paths
      - run:
          name: Install Python Development Dependencies with Pip
-          command: pip3 install --user -e .[dev]
+          command: python3 -m pip install --user -e .[dev]
      - run:
          name: Check that pip installed package is importable
-          command: python -c "import nucypher"
+          command: python3 -c "import nucypher"
      - run:
          name: Install Solidity Compiler
          command: python3 ./scripts/installation/install_solc.py
@ -459,10 +481,9 @@ commands:
  save_dependency_cache:
    description: "Cache python installation files"
    steps:
-      - run: sudo chown -R circleci:circleci /usr/local/bin
-      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
+      - chown_user_paths
      - save_cache:
-          key: pip-v9-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
+          key: pip-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
          paths:
            - "~/.local/bin"
            - "~/.local/lib/python3.7/site-packages"
@ -472,12 +493,10 @@ commands:
            - "~/.solcx/"

  restore_dependency_cache:
-    description: "Own and restore cached python installation files"
+    description: "Restore cached python installation files"
    steps:
-      - run: sudo chown -R circleci:circleci /usr/local/bin
-      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
      - restore_cache:  # ensure this step occurs *before* installing dependencies
-          key: pip-v9-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
+          key: pip-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
      - restore_cache:
          key: solc-v2-{{ checksum "nucypher/blockchain/eth/sol/__conf__.py" }}

@ -497,7 +516,7 @@ commands:
          name: Running Test Suite
          command: |
            cat test-names.tmp | tr '\n' ' ' > test-names.txt
-            pytest -c .circleci/pytest.ini `cat test-names.txt`
+            python3 -m pytest -c .circleci/pytest.ini `cat test-names.txt`

  prepare_dev_docker:
    description: "access pre-build docker image"
@ -548,8 +567,6 @@ jobs:
    <<: *python_36_base
    steps:
      - checkout
-      - run: sudo chown -R circleci:circleci /usr/local/bin
-      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.6/site-packages
      - pip_install

  pipenv_install_36:
@ -564,7 +581,6 @@ jobs:
    <<: *python_37_base
    steps:
      - checkout
-      - restore_dependency_cache  # Depends on above checkout
      - pip_install
      - save_dependency_cache

@ -579,8 +595,6 @@ jobs:
    <<: *python_38_base
    steps:
      - checkout
-      - run: sudo chown -R circleci:circleci /usr/local/bin
-      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.8/site-packages
      - pip_install

  pipenv_install_38:
@ -594,8 +608,6 @@ jobs:
    <<: *python_39_base
    steps:
    - checkout
-    - run: sudo chown -R circleci:circleci /usr/local/bin
-    - run: sudo chown -R circleci:circleci /usr/local/lib/python3.9/site-packages
    - pip_install

  pipenv_install_39:
@ -793,12 +805,13 @@ jobs:
    <<: *python_37_base
    steps:
      - prepare_environment
+      - chown_system_paths
      - run:
          name: Install Nucypher
-          command: pip3 install --user -e .[benchmark]
+          command: python3 -m pip install --user -e .[benchmark]
      - run:
          name: Estimate Gas
-          command: python tests/metrics/estimate_gas.py
+          command: python3 tests/metrics/estimate_gas.py
      - store_artifacts:
          path: tests/metrics/results/

@ -806,12 +819,10 @@ jobs:
    <<: *python_37_base
    steps:
      - prepare_environment
+      - chown_system_paths
      - run:
-          name: Install Documentation Build Dependencies (Workaround for problem with pip resolver. See 2493)
-          command: |
-            python -m pip install pip==20.2.4
-            pip3 --version
-            pip3 install --user -r docs-requirements.txt .
+          name: Install Documentation Build Dependencies
+          command: python3 -m pip install --user .[docs]
      - run:
          name: Build Sphinx Documentation
          command: make docs
@ -823,8 +834,8 @@ jobs:
    steps:
      - checkout
      - run:
-          name: Install Twine
-          command: pip3 install --user twine
+          name: Install Build Dependencies
+          command: python3 -m pip install --user .[deploy]
      - run:
          name: Build Python Distribution
          command: make dist
@ -896,8 +907,8 @@ jobs:
    steps:
      - checkout
      - run:
-          name: Install Twine
-          command: pip3 install --user twine
+          name: Install Twine and Wheel
+          command: pip3 install twine wheel
      - run:
          name: Verify git tag == __version__
          command: python3 setup.py verify
--- a/.circleci/docker/Dockerfile
+++ b/.circleci/docker/Dockerfile
@ -0,0 +1,47 @@
+FROM rust:slim-buster
+
+# prepare container
+RUN apt-get update -y
+RUN apt-get install -y sudo
+
+# setup user
+ARG USER
+RUN echo "Creating user $USER"
+RUN adduser --disabled-password --gecos '' $USER
+RUN adduser $USER sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+RUN chown -R $USER /home/$USER
+
+# install python build dependencies
+RUN sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
+    libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+    xz-utils tk-dev libffi-dev liblzma-dev git docker-compose
+
+# get python
+ARG VERSION
+RUN echo "Building python version $VERSION"
+RUN wget https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tgz
+RUN tar xzvf Python-$VERSION.tgz
+
+# build python
+WORKDIR Python-$VERSION
+RUN ./configure --enable-optimizations
+RUN make
+RUN make install
+
+# setup python
+RUN cp python /bin
+RUN apt-get install -y python3-openssl python3-dev
+
+# setup rust
+ENV CARGO_ROOT /usr/local/cargo
+ENV PATH $CARGO_ROOT/bin:$PATH
+
+# echo python and rust versions
+RUN python3 --version && \
+    cargo --version &&   \
+    rustup --version &&  \
+    rustc --version
+
+# set default user
+USER $USER
--- a/.circleci/docker/README.md
+++ b/.circleci/docker/README.md
@ -0,0 +1,26 @@
+# Rust-Python CircleCI Container
+
+
+### Setup
+```bash
+# the default username in the container
+# circle ci expects the user "circleci"
+export DOCKER_USER=<USERNAME>
+
+# the version of python to build
+export PYTHON_VERSION=<VERSION>
+```
+
+### Build
+```bash
+# pass local env vars as build args USER and VERSION
+docker build -t nucypher/rust-python:$PYTHON_VERSION . \
+--build-arg VERSION=${PYTHON_VERSION} \
+--build-arg USER=${DOCKER_USER}
+```
+
+### Push
+```bash
+# docker login or authentication required
+docker push nucypher/rust-python:$PYTHON_VERSION
+```
--- a/4
+++ b/4
@ -32,8 +32,8 @@ build-docs:

 validate-docs: build-docs
    # Requires dependencies from docs-requirements.txt
-	python newsfragments/validate_files.py
-	towncrier --draft
+	python3 newsfragments/validate_files.py
+	python3 -m towncrier --draft

 docs: build-docs validate-docs
 	readlink -f docs/build/html/index.html
--- a/setup.py
+++ b/setup.py
@ -122,6 +122,7 @@ def read_requirements(path):

 INSTALL_REQUIRES = read_requirements('requirements.txt')
 DEV_REQUIRES = read_requirements('dev-requirements.txt')
+DOC_REQUIRES = read_requirements('docs-requirements.txt')

 BENCHMARK_REQUIRES = [
    'pytest-benchmark'
@ -130,7 +131,8 @@ BENCHMARK_REQUIRES = [
 DEPLOY_REQUIRES = [
    'bumpversion',
    'ansible',
-    'twine'
+    'twine',
+    'wheel'
 ]

 URSULA_REQUIRES = ['prometheus_client', 'sentry-sdk']  # TODO: Consider renaming to 'monitor', etc.
@ -144,6 +146,7 @@ EXTRAS = {
    'dev': DEV_REQUIRES + URSULA_REQUIRES + ALICE_REQUIRES + PORTER_REQUIRES,
    'benchmark': DEV_REQUIRES + BENCHMARK_REQUIRES,
    'deploy': DEPLOY_REQUIRES,
+    'docs': DOC_REQUIRES,

    # User
    'ursula': URSULA_REQUIRES,