Nucypher
/
nucypher
mirror of https://github.com/nucypher/nucypher.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Pre-arranged merge commit between @kprasch and myself to reconcile our branches. 

Quite a few conflicts resolved.
pull/574/head
jMyles 2018-12-06 20:55:09 -05:00
parent 6a5eeab7ce 164d499884
commit a3fb853ffa
59 changed files with 3292 additions and 2390 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

273
.circleci/config.yml
View File

@ -12,7 +12,11 @@ workflows:
filters:
tags:
only: /.*/
- eth_contract_unit:
- mypy:
filters:
tags:
only: /.*/
- contracts:
context: "NuCypher Tests"
filters:
tags:
@ -20,7 +24,14 @@ workflows:
requires:
- pip_install
- pipenv_install
- config_unit:
- config:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- contracts
- crypto:
context: "NuCypher Tests"
filters:
tags:
@ -28,7 +39,7 @@ workflows:
requires:
- pip_install
- pipenv_install
- crypto_unit:
- network:
context: "NuCypher Tests"
filters:
tags:
@ -36,7 +47,7 @@ workflows:
requires:
- pip_install
- pipenv_install
- network_unit:
- keystore:
context: "NuCypher Tests"
filters:
tags:
@ -44,23 +55,7 @@ workflows:
requires:
- pip_install
- pipenv_install
- keystore_unit:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install
- pipenv_install
- blockchain_interface_unit:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- pip_install
- pipenv_install
- blockchain_entities:
- blockchain:
context: "NuCypher Tests"
filters:
tags:
@ -74,66 +69,102 @@ workflows:
tags:
only: /.*/
requires:
- pip_install
- pipenv_install
- intercontract_integration:
- crypto
- network
- keystore
- agents:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- eth_contract_unit
- mypy_type_check:
filters:
tags:
only: /.*/
requires:
- config_unit
- crypto_unit
- network_unit
- keystore_unit
- character
- cli_tests:
- blockchain
- contracts
- actors:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- blockchain_entities
- blockchain_interface_unit
- config_unit
- crypto_unit
- network_unit
- keystore_unit
- blockchain
- contracts
- deployers:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- blockchain
- contracts
- config:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- blockchain
- crypto
- network
- keystore
- cli:
context: "NuCypher Tests"
filters:
tags:
only: /.*/
requires:
- actors
- deployers
- config
- character
- test_deploy:
context: "NuCypher PyPI"
requires:
- mypy_type_check
- cli_tests
- ursula_command:
context: "NuCypher Tests"
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
- request_publication_approval:
type: approval
only: /.*/
requires:
- test_deploy
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
- deploy:
context: "NuCypher PyPI"
requires:
- request_publication_approval
filters:
tags:
only: /v[0-9]+.*/
branches:
ignore: /.*/
- actors
- deployers
- config
- character
#
# TODO: Initial Publication Automation
#
# - test_build:
# filters:
# tags:
# only: /.*/
# requires:
# - cli
# - ursula_command
# - test_deploy:
# context: "NuCypher PyPI"
# requires:
# - test_build
# filters:
# tags:
# only: /v[0-9]+.*/
# branches:
# ignore: /.*/
# - request_publication_approval:
# type: approval
# requires:
# - test_deploy
# filters:
# tags:
# only: /v[0-9]+.*/
# branches:
# ignore: /.*/
# - deploy:
# context: "NuCypher PyPI"
# requires:
# - request_publication_approval
# filters:
# tags:
# only: /v[0-9]+.*/
# branches:
# ignore: /.*/
#
python_36_base: &python_36_base
@ -187,7 +218,7 @@ jobs:
name: Check Python Entrypoint
command: python3 -c "import nucypher; print(nucypher.__version__)"
blockchain_interface_unit:
blockchain:
<<: *python_36_base
parallelism: 2
steps:
@ -196,30 +227,56 @@ jobs:
at: ~/.local/share/virtualenvs/
- run:
name: Blockchain Interface Tests
command: |
pipenv run pytest --cov=nucypher/blockchain/eth -v --runslow $(circleci tests glob tests/blockchain/eth/interfaces/**/test_*.py | circleci tests split --split-by=timings) --junitxml=./reports/pytest/blockchain_interface_results.xml
command: pipenv run pytest --cov=nucypher/blockchain/eth -v --runslow tests/blockchain/eth/interfaces --junitxml=./reports/pytest/results.xml
- run: *coveralls
- store_test_results:
path: ./reports/pytest/
blockchain_entities:
agents:
<<: *python_36_base
parallelism: 6
parallelism: 2
steps:
- checkout
- attach_workspace:
at: ~/.local/share/virtualenvs/
- run:
name: Blockchain Interface Tests
command: |
pipenv run pytest --cov=nucypher/blockchain/eth -v --runslow $(circleci tests glob tests/blockchain/eth/entities/**/test_*.py | circleci tests split --split-by=timings) --junitxml=./reports/pytest/blockchain_interface_results.xml
name: Blockchain Agent Tests
command: pipenv run pytest --cov=nucypher/blockchain/eth -v --runslow $(circleci tests glob tests/blockchain/eth/entities/agents/**/*.py | circleci tests split --split-by=timings) --junitxml=./reports/pytest/results.xml
- run: *coveralls
- store_test_results:
path: ./reports/pytest/
eth_contract_unit:
actors:
<<: *python_36_base
parallelism: 2
steps:
- checkout
- attach_workspace:
at: ~/.local/share/virtualenvs/
- run:
name: Blockchain Actor Tests
command: pipenv run pytest --cov=nucypher/blockchain/eth -v --runslow $(circleci tests glob tests/blockchain/eth/entities/actors/**/*.py | circleci tests split --split-by=timings) --junitxml=./reports/pytest/results.xml
- run: *coveralls
- store_test_results:
path: ./reports/pytest/
deployers:
<<: *python_36_base
parallelism: 4
steps:
- checkout
- attach_workspace:
at: ~/.local/share/virtualenvs/
- run:
name: Contract Deployer Tests
command: pipenv run pytest --cov=nucypher/blockchain/eth -v --runslow --junitxml=./reports/pytest/results.xml $(circleci tests glob tests/blockchain/eth/entities/deployers/test_*.py | circleci tests split --split-by=timings)
- run: *coveralls
- store_test_results:
path: ./reports/pytest/
contracts:
<<: *python_36_base
parallelism: 5
steps:
- checkout
- attach_workspace:
@ -228,11 +285,10 @@ jobs:
name: Ethereum Contract Unit Tests
command: |
pipenv run pytest --junitxml=./reports/pytest/eth-contract-unit-report.xml -v --runslow $(circleci tests glob tests/blockchain/eth/contracts/**/**/test_*.py | circleci tests split --split-by=timings)
- run: *coveralls
- store_test_results:
path: ./reports/pytest/
config_unit:
config:
<<: *python_36_base
parallelism: 2
steps:
@ -247,7 +303,7 @@ jobs:
- store_test_results:
path: ./reports/pytest/
crypto_unit:
crypto:
<<: *python_36_base
steps:
- checkout
@ -261,7 +317,7 @@ jobs:
- store_test_results:
path: ./reports/pytest/
network_unit:
network:
<<: *python_36_base
steps:
- checkout
@ -275,7 +331,7 @@ jobs:
- store_test_results:
path: ./reports/pytest/
keystore_unit:
keystore:
<<: *python_36_base
parallelism: 2
steps:
@ -305,20 +361,20 @@ jobs:
- store_test_results:
path: ./reports/pytest/
intercontract_integration:
learning:
<<: *python_36_base
steps:
- checkout
- attach_workspace:
at: ~/.local/share/virtualenvs/
- run:
name: Ethereum Inter-Contract Integration Test
command: |
pipenv run pytest -v --runslow tests/blockchain/eth/contracts/main/test_intercontract_integration.py --junitxml=./reports/pytest/intercontract_integration_results.xml
name: Learner Tests
command: pipenv run pytest --cov=nucypher -v --runslow tests/learning --junitxml=./reports/pytest/results.xml
- run: *coveralls
- store_test_results:
path: ./reports/pytest/
cli_tests:
cli:
<<: *python_36_base
parallelism: 4
steps:
@ -328,21 +384,34 @@ jobs:
- run:
name: Nucypher CLI Tests
command: |
pipenv run pytest --cov=nucypher/cli.py -v --runslow $(circleci tests glob tests/cli/**/test_*.py | circleci tests split --split-by=timings) --junitxml=./reports/pytest/cli_results.xml
- run: *coveralls
pipenv run pytest -v --runslow $(circleci tests glob tests/cli/**/test_*.py | circleci tests split --split-by=timings) --junitxml=./reports/pytest/cli_results.xml
- store_test_results:
path: ./reports/pytest/
mypy_type_check:
ursula_command:
<<: *python_36_base
steps:
- checkout
- attach_workspace:
at: ~/.local/share/virtualenvs/
- run:
name: Ursula Command Tests
command: |
pipenv run pytest -v --runslow tests/cli/protocol --junitxml=./reports/pytest/results.xml
- store_test_results:
path: ./reports/pytest/
mypy:
<<: *python_36_base
steps:
- checkout
- attach_workspace:
at: ~/.local/share/virtualenvs/
- run:
name: Install lxml
name: Install mypy
command: |
pipenv run pip install lxml
pipenv run pip install mypy
- run:
name: Run Mypy Static Type Checks (Always Succeed)
command: |
@ -352,6 +421,30 @@ jobs:
- store_artifacts:
path: ./mypy_reports
test_build:
<<: *python_36_base
steps:
- checkout
- run:
name: Install Dependencies (Test Build)
command: |
pipenv install --three --dev --skip-lock --pre
pipenv install --dev --skip-lock twine
- run:
name: Build Python Wheel
command: |
pipenv run python setup.py sdist
pipenv run python setup.py bdist_wheel -v
- run:
name: Install Nucypher via Wheel
command: |
pip3 install --user ./dist/nucypher-0.1.0a0-py3-none-any.whl[test]
- run:
name: Run Entrypoint Version Commands
command: |
python -c "import nucypher; print(nucypher.__version__)"
nucypher --version
test_deploy:
<<: *python_36_base
steps:

1
Pipfile
View File

@ -59,6 +59,7 @@ python-coveralls = "*"
ansible = "*"
moto = "*"
nucypher = {editable = true, path = "."}
pytest-mock = "*"
[scripts]
install-solc = "./scripts/install_solc.sh"

10
examples/finnegans_wake_demo/run_federated_ursula.py
View File

@ -32,16 +32,16 @@ MY_REST_PORT = sys.argv[1]
# TODO: Use real path tooling here.
SHARED_CRUFTSPACE = "{}/examples-runtime-cruft".format(os.path.dirname(os.path.abspath(__file__)))
CRUFTSPACE = "{}/{}".format(SHARED_CRUFTSPACE, MY_REST_PORT)
DB_NAME = "{}/database".format(CRUFTSPACE)
db_filepath = "{}/database".format(CRUFTSPACE)
CERTIFICATE_DIR = "{}/certs".format(CRUFTSPACE)
def spin_up_ursula(rest_port, db_name, teachers=(), certificate_dir=None):
def spin_up_ursula(rest_port, db_filepath, teachers=(), certificate_dir=None):
metadata_file = "examples-runtime-cruft/node-metadata-{}".format(rest_port)
_URSULA = Ursula(rest_port=rest_port,
rest_host="localhost",
db_name=db_name,
db_filepath=db_filepath,
federated_only=True,
known_nodes=teachers,
known_certificates_dir=certificate_dir
@ -52,7 +52,7 @@ def spin_up_ursula(rest_port, db_name, teachers=(), certificate_dir=None):
_URSULA.start_learning_loop()
_URSULA.get_deployer().run()
finally:
os.remove(db_name)
os.remove(db_filepath)
os.remove(metadata_file)
@ -78,7 +78,7 @@ if __name__ == "__main__":
except FileNotFoundError as e:
raise ValueError("Can't find a metadata file for node {}".format(teacher_rest_port))
spin_up_ursula(MY_REST_PORT, DB_NAME,
spin_up_ursula(MY_REST_PORT, db_filepath,
teachers=teachers,
certificate_dir=CERTIFICATE_DIR)
finally:

2
nucypher/__about__.py
View File

@ -14,6 +14,8 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import absolute_import, division, print_function
__all__ = [

6
nucypher/blockchain/eth/registry.py
View File

@ -14,6 +14,8 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import json
import os
import tempfile
@ -189,7 +191,7 @@ class TemporaryEthereumContractRegistry(EthereumContractRegistry):
class InMemoryEthereumContractRegistry(EthereumContractRegistry):
def __init__(self) -> None:
super().__init__(registry_filepath=":memory:")
super().__init__(registry_filepath="::memory-registry::")
self.__registry_data = None # type: str
def clear(self):
@ -277,7 +279,7 @@ class AllocationRegistry(EthereumContractRegistry):
class InMemoryAllocationRegistry(AllocationRegistry):
def __init__(self, *args, **kwargs) -> None:
super().__init__(registry_filepath=":memory:", *args, **kwargs)
super().__init__(registry_filepath="::memory-registry::", *args, **kwargs)
self.__registry_data = None # type: str
def clear(self):

73
nucypher/characters/base.py
View File

@ -16,10 +16,6 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from contextlib import suppress
from typing import Dict, ClassVar, Set
from typing import Optional
from typing import Tuple
from typing import Union, List
from eth_keys import KeyAPI as EthKeyAPI
from eth_utils import to_checksum_address, to_canonical_address
@ -27,6 +23,24 @@ from umbral.keys import UmbralPublicKey
from umbral.signing import Signature
from constant_sorrow import constants, default_constant_splitter
from eth_keys import KeyAPI as EthKeyAPI
from eth_utils import to_checksum_address, to_canonical_address
from typing import Dict, ClassVar
from typing import Optional
from typing import Tuple
from typing import Union, List
from constant_sorrow import default_constant_splitter
from constant_sorrow.constants import (
NO_NICKNAME,
NO_BLOCKCHAIN_CONNECTION,
STRANGER,
NO_SIGNING_POWER,
DO_NOT_SIGN,
NO_DECRYPTION_PERFORMED,
SIGNATURE_TO_FOLLOW,
SIGNATURE_IS_ON_CIPHERTEXT
)
from nucypher.blockchain.eth.chains import Blockchain
from nucypher.crypto.api import encrypt_and_sign
from nucypher.crypto.kits import UmbralMessageKit
@ -42,6 +56,8 @@ from nucypher.crypto.signing import signature_splitter, StrangerStamp, Signature
from nucypher.network.middleware import RestMiddleware
from nucypher.network.nicknames import nickname_from_seed
from nucypher.network.nodes import Learner
from umbral.keys import UmbralPublicKey
from umbral.signing import Signature
class Character(Learner):
@ -61,7 +77,7 @@ class Character(Learner):
is_me: bool = True,
federated_only: bool = False,
blockchain: Blockchain = None,
checksum_address: bytes = constants.NO_BLOCKCHAIN_CONNECTION.bool_value(False),
checksum_public_address: bytes = NO_BLOCKCHAIN_CONNECTION.bool_value(False),
network_middleware: RestMiddleware = None,
keyring_dir: str = None,
crypto_power: CryptoPower = None,
@ -109,7 +125,7 @@ class Character(Learner):
else:
self._crypto_power = CryptoPower(power_ups=self._default_crypto_powerups)
self._checksum_address = checksum_address
self._checksum_address = checksum_public_address
#
# Self-Character
#
@ -128,7 +144,7 @@ class Character(Learner):
signing_power = self._crypto_power.power_ups(SigningPower) # type: SigningPower
self._stamp = signing_power.get_signature_stamp() # type: SignatureStamp
except NoSigningPower:
self._stamp = constants.NO_SIGNING_POWER
self._stamp = NO_SIGNING_POWER
#
# Learner
@ -145,17 +161,18 @@ class Character(Learner):
if network_middleware is not None:
raise TypeError("Network middleware cannot be attached to a Stanger-Character.")
self._stamp = StrangerStamp(self.public_keys(SigningPower))
self.keyring_dir = constants.STRANGER
self.network_middleware = constants.STRANGER
self.keyring_dir = STRANGER
self.network_middleware = STRANGER
#
# Decentralized
#
if not federated_only:
if not checksum_address:
raise ValueError("No checksum_address provided while running in a non-federated mode.")
if not checksum_public_address:
raise ValueError("No checksum_public_address provided while running in a non-federated mode.")
else:
self._checksum_address = checksum_address # TODO: Check that this matches BlockchainPower
self._checksum_address = checksum_public_address # TODO: Check that this matches BlockchainPower
#
# Federated
#
@ -163,21 +180,27 @@ class Character(Learner):
try:
self._set_checksum_address() # type: str
except NoSigningPower:
self._checksum_address = constants.NO_BLOCKCHAIN_CONNECTION
if checksum_address:
self._checksum_address = NO_BLOCKCHAIN_CONNECTION
if checksum_public_address:
# We'll take a checksum address, as long as it matches their singing key
if not checksum_address == self.checksum_public_address:
if not checksum_public_address == self.checksum_public_address:
error = "Federated-only Characters derive their address from their Signing key; got {} instead."
raise self.SuspiciousActivity(error.format(checksum_address))
raise self.SuspiciousActivity(error.format(checksum_public_address))
#
# Nicknames
#
try:
self.nickname, self.nickname_metadata = nickname_from_seed(self.checksum_public_address)
except SigningPower.not_found_error:
if self.federated_only:
self.nickname = self.nickname_metadata = constants.NO_NICKNAME
self.nickname = self.nickname_metadata = NO_NICKNAME
else:
raise
#
# Fleet state
#
if is_me is True:
self.known_nodes.record_fleet_state()
@ -202,7 +225,7 @@ class Character(Learner):
@property
def stamp(self):
if self._stamp is constants.NO_SIGNING_POWER:
if self._stamp is NO_SIGNING_POWER:
raise NoSigningPower
elif not self._stamp:
raise AttributeError("SignatureStamp has not been set up yet.")
@ -219,7 +242,7 @@ class Character(Learner):
@property
def checksum_public_address(self):
if self._checksum_address is constants.NO_BLOCKCHAIN_CONNECTION:
if self._checksum_address is NO_BLOCKCHAIN_CONNECTION:
self._set_checksum_address()
return self._checksum_address
@ -239,7 +262,7 @@ class Character(Learner):
with the public_material_bytes, and the resulting CryptoPowerUp instance
consumed by the Character.
# TODO: Need to be federated only until we figure out the best way to get the checksum_address in here.
# TODO: Need to be federated only until we figure out the best way to get the checksum_public_address in here.
"""
@ -274,7 +297,7 @@ class Character(Learner):
:return: A tuple, (ciphertext, signature). If sign==False,
then signature will be NOT_SIGNED.
"""
signer = self.stamp if sign else constants.DO_NOT_SIGN
signer = self.stamp if sign else DO_NOT_SIGN
message_kit, signature = encrypt_and_sign(recipient_pubkey_enc=recipient.public_keys(EncryptingPower),
plaintext=plaintext,
@ -323,12 +346,12 @@ class Character(Learner):
cleartext_with_sig_header = self.decrypt(message_kit=message_kit,
label=label)
sig_header, cleartext = default_constant_splitter(cleartext_with_sig_header, return_remainder=True)
if sig_header == constants.SIGNATURE_IS_ON_CIPHERTEXT:
# The ciphertext is what is signed - note that for later.
if sig_header == SIGNATURE_IS_ON_CIPHERTEXT:
# THe ciphertext is what is signed - note that for later.
message = message_kit.ciphertext
if not signature:
raise ValueError("Can't check a signature on the ciphertext if don't provide one.")
elif sig_header == constants.SIGNATURE_TO_FOLLOW:
elif sig_header == SIGNATURE_TO_FOLLOW:
# The signature follows in this cleartext - split it off.
signature_from_kit, cleartext = signature_splitter(cleartext,
return_remainder=True)
@ -336,7 +359,7 @@ class Character(Learner):
else:
# Not decrypting - the message is the object passed in as a message kit. Cast it.
message = bytes(message_kit)
cleartext = constants.NO_DECRYPTION_PERFORMED
cleartext = NO_DECRYPTION_PERFORMED
if signature and signature_from_kit:
if signature != signature_from_kit:

45
nucypher/characters/lawful.py
View File

@ -24,23 +24,30 @@ from typing import Set
import maya
import requests
import socket
import time
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurve
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.x509 import load_pem_x509_certificate, Certificate
from cryptography.x509 import load_pem_x509_certificate, Certificate, NameOID
from eth_utils import to_checksum_address
from functools import partial
from twisted.internet import threads
from umbral.keys import UmbralPublicKey
from umbral.signing import Signature
from typing import Dict
from typing import Iterable
from typing import List
from bytestring_splitter import VariableLengthBytestring, BytestringKwargifier, BytestringSplitter, \
BytestringSplittingError
from constant_sorrow import constants
from constant_sorrow.constants import INCLUDED_IN_BYTESTRING, constant_or_bytes
from bytestring_splitter import BytestringSplitter, VariableLengthBytestring
from constant_sorrow import constants
from constant_sorrow.constants import PUBLIC_ONLY
from nucypher.blockchain.eth.actors import PolicyAuthor, Miner
from nucypher.blockchain.eth.agents import MinerAgent
from nucypher.characters.base import Character, Learner
from nucypher.config.storages import NodeStorage
from nucypher.config.storages import NodeStorage, ForgetfulNodeStorage
from nucypher.crypto.api import keccak_digest
from nucypher.crypto.constants import PUBLIC_KEY_LENGTH, PUBLIC_ADDRESS_LENGTH
from nucypher.crypto.powers import SigningPower, EncryptingPower, DelegatingPower, BlockchainPower
@ -48,7 +55,13 @@ from nucypher.keystore.keypairs import HostingKeypair
from nucypher.network.nicknames import nickname_from_seed
from nucypher.network.nodes import Teacher
from nucypher.network.protocols import InterfaceInfo
from nucypher.network.middleware import RestMiddleware
from nucypher.network.nodes import Teacher
from nucypher.network.protocols import InterfaceInfo, parse_node_uri
from nucypher.network.server import ProxyRESTServer, TLSHostingPower, ProxyRESTRoutes
from nucypher.utilities.decorators import validate_checksum_address
from umbral.keys import UmbralPublicKey
from umbral.signing import Signature
class Alice(Character, PolicyAuthor):
@ -57,11 +70,11 @@ class Alice(Character, PolicyAuthor):
def __init__(self, is_me=True, federated_only=False, network_middleware=None, *args, **kwargs) -> None:
policy_agent = kwargs.pop("policy_agent", None)
checksum_address = kwargs.pop("checksum_address", None)
checksum_address = kwargs.pop("checksum_public_address", None)
Character.__init__(self,
is_me=is_me,
federated_only=federated_only,
checksum_address=checksum_address,
checksum_public_address=checksum_address,
network_middleware=network_middleware,
*args, **kwargs)
@ -177,8 +190,8 @@ class Alice(Character, PolicyAuthor):
# Wait for a revocation threshold of nodes to be known ((n - m) + 1)
revocation_threshold = ((policy.n - policy.treasure_map.m) + 1)
self.block_until_specific_nodes_are_known(
policy.revocation_kit.revokable_addresses,
allow_missing=(policy.n - revocation_threshold))
policy.revocation_kit.revokable_addresses,
allow_missing=(policy.n - revocation_threshold))
except self.NotEnoughTeachers as e:
raise e
else:
@ -445,19 +458,17 @@ class Ursula(Teacher, Character, Miner):
domains: Set = (constants.GLOBAL_DOMAIN,), # For now, serving and learning domains will be the same.
certificate: Certificate = None,
certificate_filepath: str = None,
db_name: str = None,
db_filepath: str = None,
is_me: bool = True,
interface_signature=None,
timestamp=None,
# Blockchain
checksum_address: str = None,
identity_evidence: bytes = constants.NOT_SIGNED,
checksum_public_address: str = None,
# Character
passphrase: str = None,
password: str = None,
abort_on_learning_error: bool = False,
federated_only: bool = False,
start_learning_now: bool = None,
@ -474,7 +485,7 @@ class Ursula(Teacher, Character, Miner):
self._work_orders = list()
Character.__init__(self,
is_me=is_me,
checksum_address=checksum_address,
checksum_public_address=checksum_public_address,
start_learning_now=start_learning_now,
federated_only=federated_only,
crypto_power=crypto_power,
@ -493,12 +504,15 @@ class Ursula(Teacher, Character, Miner):
# Staking Ursula
#
if not federated_only:
Miner.__init__(self, is_me=is_me, checksum_address=checksum_address)
Miner.__init__(self, is_me=is_me, checksum_address=checksum_public_address)
# Access staking node via node's transacting keys TODO: Better handle ephemeral staking self ursula
blockchain_power = BlockchainPower(blockchain=self.blockchain, account=self.checksum_public_address)
self._crypto_power.consume_power_up(blockchain_power)
# Use blockchain power to substantiate stamp, instead of signing key
self.substantiate_stamp(password=password) # TODO: Derive from keyring
#
# ProxyRESTServer and TLSHostingPower # TODO: Maybe we want _power_ups to be public after all?
#
@ -514,7 +528,6 @@ class Ursula(Teacher, Character, Miner):
# REST Server (Ephemeral Self-Ursula)
#
rest_routes = ProxyRESTRoutes(
db_name=db_name,
db_filepath=db_filepath,
network_middleware=self.network_middleware,
federated_only=self.federated_only, # TODO: 466
@ -578,7 +591,7 @@ class Ursula(Teacher, Character, Miner):
timestamp=timestamp,
identity_evidence=identity_evidence,
substantiate_immediately=is_me and not federated_only,
passphrase=passphrase)
)
#
# Logging / Updating

13
nucypher/characters/unlawful.py
View File

@ -18,7 +18,7 @@ from eth_tester.exceptions import ValidationError
from nucypher.characters.lawful import Ursula
from nucypher.crypto.powers import CryptoPower, SigningPower
from nucypher.utilities.sandbox.constants import TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
from nucypher.utilities.sandbox.constants import INSECURE_DEVELOPMENT_PASSWORD, MOCK_URSULA_DB_FILEPATH
from nucypher.utilities.sandbox.middleware import EvilMiddleWare
@ -30,7 +30,7 @@ class Vladimir(Ursula):
network_middleware = EvilMiddleWare()
fraud_address = '0xbad022A87Df21E4c787C7B1effD5077014b8CC45'
fraud_key = 'a75d701cc4199f7646909d15f22e2e0ef6094b3e2aa47a188f35f47e8932a7b9'
db_name = 'vladimir.db'
db_filepath = MOCK_URSULA_DB_FILEPATH
@classmethod
def from_target_ursula(cls,
@ -53,13 +53,12 @@ class Vladimir(Ursula):
vladimir = cls(is_me=True,
crypto_power=crypto_power,
db_name=cls.db_name,
db_filepath=cls.db_name,
db_filepath=cls.db_filepath,
rest_host=target_ursula.rest_information()[0].host,
rest_port=target_ursula.rest_information()[0].port,
certificate=target_ursula.rest_server_certificate(),
network_middleware=cls.network_middleware,
checksum_address = cls.fraud_address,
checksum_public_address = cls.fraud_address,
######### Asshole.
timestamp=target_ursula._timestamp,
interface_signature=target_ursula._interface_signature_object,
@ -76,8 +75,8 @@ class Vladimir(Ursula):
Upload Vladimir's ETH keys to the keychain via web3 / RPC.
"""
try:
passphrase = TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
blockchain.interface.w3.personal.importRawKey(private_key=cls.fraud_key, passphrase=passphrase)
password = INSECURE_DEVELOPMENT_PASSWORD
blockchain.interface.w3.personal.importRawKey(private_key=cls.fraud_key, passphrase=password)
except (ValidationError, ):
# check if Vlad's key is already on the keyring...
if cls.fraud_address in blockchain.interface.w3.personal.listAccounts:

1152
nucypher/cli.py
View File

File diff suppressed because it is too large Load Diff

0
tests/learning/__init__.py → nucypher/cli/__init__.py
View File

223
nucypher/cli/deploy.py Normal file
View File

@ -0,0 +1,223 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import collections
import hashlib
import json
import click
from twisted.logger import Logger
from twisted.logger import globalLogPublisher
from typing import ClassVar, Tuple
from nucypher.blockchain.eth.agents import EthereumContractAgent
from nucypher.blockchain.eth.deployers import (
NucypherTokenDeployer,
MinerEscrowDeployer,
PolicyManagerDeployer,
ContractDeployer
)
from nucypher.cli.painting import BANNER
from nucypher.cli.types import EIP55_CHECKSUM_ADDRESS
from nucypher.config.node import NodeConfiguration
from nucypher.utilities.logging import getTextFileObserver
#
# Click Eager Functions
#
def echo_version(ctx, param, value):
if not value or ctx.resilient_parsing:
return
click.secho(BANNER, bold=True)
ctx.exit()
#
# Deployers
#
DeployerInfo = collections.namedtuple('DeployerInfo', ('deployer_class', # type: ContractDeployer
'upgradeable', # type: bool
'agent_name', # type: EthereumContractAgent
'dependant')) # type: EthereumContractAgent
DEPLOYERS = collections.OrderedDict({
NucypherTokenDeployer._contract_name: DeployerInfo(deployer_class=NucypherTokenDeployer,
upgradeable=False,
agent_name='token_agent',
dependant=None),
MinerEscrowDeployer._contract_name: DeployerInfo(deployer_class=MinerEscrowDeployer,
upgradeable=True,
agent_name='miner_agent',
dependant='token_agent'),
PolicyManagerDeployer._contract_name: DeployerInfo(deployer_class=PolicyManagerDeployer,
upgradeable=True,
agent_name='policy_agent',
dependant='miner_agent')
})
class NucypherDeployerClickConfig:
log_to_file = True # TODO: Use envvar
def __init__(self):
if self.log_to_file is True:
globalLogPublisher.addObserver(getTextFileObserver())
self.log = Logger(self.__class__.__name__)
# Register the above class as a decorator
nucypher_deployer_config = click.make_pass_decorator(NucypherDeployerClickConfig, ensure=True)
@click.command()
@click.argument('action')
@click.option('--contract-name', help="Deploy a single contract by name", type=click.STRING)
@click.option('--force', is_flag=True)
@click.option('--deployer-address', help="Deployer's checksum address", type=EIP55_CHECKSUM_ADDRESS)
@click.option('--registry-outfile', help="Output path for new registry", type=click.Path(), default=NodeConfiguration.REGISTRY_SOURCE)
@nucypher_deployer_config
def deploy(config,
action,
deployer_address,
contract_name,
registry_outfile,
force):
"""Manage contract and registry deployment"""
if not config.deployer:
click.secho("The --deployer flag must be used to issue the deploy command.", fg='red', bold=True)
raise click.Abort()
def __get_deployers():
config.registry_filepath = registry_outfile
config.connect_to_blockchain()
config.blockchain.interface.deployer_address = deployer_address or config.accounts[0]
click.confirm("Continue?", abort=True)
return deployers
if action == "contracts":
deployers = __get_deployers()
__deployment_transactions = dict()
__deployment_agents = dict()
available_deployers = ", ".join(deployers)
click.echo("\n-----------------------------------------------")
click.echo("Available Deployers: {}".format(available_deployers))
click.echo("Blockchain Provider URI ... {}".format(config.blockchain.interface.provider_uri))
click.echo("Registry Output Filepath .. {}".format(config.blockchain.interface.registry.filepath))
click.echo("Deployer's Address ........ {}".format(config.blockchain.interface.deployer_address))
click.echo("-----------------------------------------------\n")
def __deploy_contract(deployer_class: ClassVar,
upgradeable: bool,
agent_name: str,
dependant: str = None
) -> Tuple[dict, EthereumContractAgent]:
__contract_name = deployer_class._contract_name
__deployer_init_args = dict(blockchain=config.blockchain,
deployer_address=config.blockchain.interface.deployer_address)
if dependant is not None:
__deployer_init_args.update({dependant: __deployment_agents[dependant]})
if upgradeable:
secret = click.prompt("Enter deployment secret for {}".format(__contract_name),
hide_input=True, confirmation_prompt=True)
secret_hash = hashlib.sha256(secret)
__deployer_init_args.update({'secret_hash': secret_hash})
__deployer = deployer_class(**__deployer_init_args)
#
# Arm
#
if not force:
click.confirm("Arm {}?".format(deployer_class.__name__), abort=True)
is_armed, disqualifications = __deployer.arm(abort=False)
if not is_armed:
disqualifications = ', '.join(disqualifications)
click.secho("Failed to arm {}. Disqualifications: {}".format(__contract_name, disqualifications),
fg='red', bold=True)
raise click.Abort()
#
# Deploy
#
if not force:
click.confirm("Deploy {}?".format(__contract_name), abort=True)
__transactions = __deployer.deploy()
__deployment_transactions[__contract_name] = __transactions
__agent = __deployer.make_agent()
__deployment_agents[agent_name] = __agent
click.secho("Deployed {} - Contract Address: {}".format(contract_name, __agent.contract_address),
fg='green', bold=True)
return __transactions, __agent
if contract_name:
#
# Deploy Single Contract
#
try:
deployer_info = deployers[contract_name]
except KeyError:
click.secho(
"No such contract {}. Available contracts are {}".format(contract_name, available_deployers),
fg='red', bold=True)
raise click.Abort()
else:
_txs, _agent = __deploy_contract(deployer_info.deployer_class,
upgradeable=deployer_info.upgradeable,
agent_name=deployer_info.agent_name,
dependant=deployer_info.dependant)
else:
#
# Deploy All Contracts
#
for deployer_name, deployer_info in deployers.items():
_txs, _agent = __deploy_contract(deployer_info.deployer_class,
upgradeable=deployer_info.upgradeable,
agent_name=deployer_info.agent_name,
dependant=deployer_info.dependant)
if not force and click.prompt("View deployment transaction hashes?"):
for contract_name, transactions in __deployment_transactions.items():
click.echo(contract_name)
for tx_name, txhash in transactions.items():
click.echo("{}:{}".format(tx_name, txhash))
if not force and click.confirm("Save transaction hashes to JSON file?"):
file = click.prompt("Enter output filepath", type=click.File(mode='w')) # TODO: Save Txhashes
file.__write(json.dumps(__deployment_transactions))
click.secho("Successfully wrote transaction hashes file to {}".format(file.path), fg='green')
else:
raise click.BadArgumentUsage

525
nucypher/cli/main.py Normal file
View File

@ -0,0 +1,525 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
import click
from nacl.exceptions import CryptoError
from twisted.internet import stdio
from twisted.logger import Logger
from twisted.logger import globalLogPublisher
from constant_sorrow.constants import NO_BLOCKCHAIN_CONNECTION, NO_PASSWORD
from nucypher.blockchain.eth.constants import MIN_LOCKED_PERIODS, MAX_MINTING_PERIODS
from nucypher.characters.lawful import Ursula
from nucypher.cli.painting import BANNER, paint_configuration, paint_known_nodes, paint_contract_status
from nucypher.cli.protocol import UrsulaCommandProtocol
from nucypher.cli.types import (
EIP55_CHECKSUM_ADDRESS,
UNREGISTERED_PORT,
EXISTING_READABLE_FILE,
EXISTING_WRITABLE_DIRECTORY,
STAKE_VALUE,
STAKE_DURATION
)
from nucypher.config.characters import UrsulaConfiguration
from nucypher.utilities.logging import (
logToSentry,
getTextFileObserver,
initialize_sentry,
getJsonFileObserver,
SimpleObserver)
#
# Click CLI Config
#
class NucypherClickConfig:
__sentry_endpoint = "https://d8af7c4d692e4692a455328a280d845e@sentry.io/1310685" # TODO: Use nucypher domain
# Environment Variables
config_file = os.environ.get('NUCYPHER_CONFIG_FILE', None)
sentry_endpoint = os.environ.get("NUCYPHER_SENTRY_DSN", __sentry_endpoint)
log_to_sentry = os.environ.get("NUCYPHER_SENTRY_LOGS", True)
log_to_file = os.environ.get("NUCYPHER_FILE_LOGS", True)
# Sentry Logging
if log_to_sentry is True:
initialize_sentry(dsn=__sentry_endpoint)
globalLogPublisher.addObserver(logToSentry)
# File Logging
if log_to_file is True:
globalLogPublisher.addObserver(getTextFileObserver())
globalLogPublisher.addObserver(getJsonFileObserver())
def __init__(self):
self.log = Logger(self.__class__.__name__)
self.__keyring_password = NO_PASSWORD
def get_password(self, confirm: bool =False) -> str:
keyring_password = os.environ.get("NUCYPHER_KEYRING_PASSWORD", NO_PASSWORD)
if keyring_password is NO_PASSWORD: # Collect password, prefer env var
prompt = "Enter keyring password"
keyring_password = click.prompt(prompt, confirmation_prompt=confirm, hide_input=True)
self.__keyring_password = keyring_password
return self.__keyring_password
# Register the above click configuration class as a decorator
nucypher_click_config = click.make_pass_decorator(NucypherClickConfig, ensure=True)
def echo_version(ctx, param, value):
if not value or ctx.resilient_parsing:
return
click.secho(BANNER, bold=True)
ctx.exit()
#
# Common CLI
#
@click.group()
@click.option('--version', help="Echo the CLI version", is_flag=True, callback=echo_version, expose_value=False, is_eager=True)
@click.option('-v', '--verbose', help="Specify verbosity level", count=True)
@nucypher_click_config
def nucypher_cli(click_config, verbose):
click.echo(BANNER)
click_config.verbose = verbose
if click_config.verbose:
click.secho("Verbose mode is enabled", fg='blue')
@nucypher_cli.command()
@click.option('--config-file', help="Path to configuration file", type=EXISTING_READABLE_FILE)
@nucypher_click_config
def status(click_config, config_file):
"""
Echo a snapshot of live network metadata.
"""
#
# Initialize
#
ursula_config = UrsulaConfiguration.from_configuration_file(filepath=config_file)
if not ursula_config.federated_only:
ursula_config.connect_to_blockchain(provider_uri=ursula_config.provider_uri)
ursula_config.connect_to_contracts()
# Contracts
paint_contract_status(ursula_config=ursula_config, click_config=click_config)
# Known Nodes
paint_known_nodes(ursula=ursula_config)
@nucypher_cli.command()
@click.argument('action')
@click.option('--debug', '-D', help="Enable debugging mode", is_flag=True)
@click.option('--dev', '-d', help="Enable development mode", is_flag=True)
@click.option('--force', '-f', help="Don't ask for confirmation", is_flag=True)
@click.option('--teacher-uri', help="An Ursula URI to start learning from (seednode)", type=click.STRING)
@click.option('--min-stake', help="The minimum stake the teacher must have to be a teacher", type=click.INT, default=0)
@click.option('--rest-host', help="The host IP address to run Ursula network services on", type=click.STRING)
@click.option('--rest-port', help="The host port to run Ursula network services on", type=UNREGISTERED_PORT)
@click.option('--db-filepath', help="The database filepath to connect to", type=click.STRING)
@click.option('--checksum-address', help="Run with a specified account", type=EIP55_CHECKSUM_ADDRESS)
@click.option('--federated-only', help="Connect only to federated nodes", is_flag=True, default=True)
@click.option('--poa', help="Inject POA middleware", is_flag=True)
@click.option('--config-root', help="Custom configuration directory", type=click.Path())
@click.option('--config-file', help="Path to configuration file", type=EXISTING_READABLE_FILE)
@click.option('--metadata-dir', help="Custom known metadata directory", type=EXISTING_WRITABLE_DIRECTORY)
@click.option('--provider-uri', help="Blockchain provider's URI", type=click.STRING)
@click.option('--no-registry', help="Skip importing the default contract registry", is_flag=True)
@click.option('--registry-filepath', help="Custom contract registry filepath", type=EXISTING_READABLE_FILE)
@nucypher_click_config
def ursula(click_config,
action,
debug,
dev,
force,
teacher_uri,
min_stake,
rest_host,
rest_port,
db_filepath,
checksum_address,
federated_only,
poa,
config_root,
config_file,
metadata_dir, # TODO: Start nodes from an additional existing metadata dir
provider_uri,
no_registry,
registry_filepath
) -> None:
"""
Manage and run an Ursula node.
\b
Actions
-------------------------------------------------
\b
run Run an "Ursula" node.
init Create a new Ursula node configuration.
view View the Ursula node's configuration.
forget Forget all known nodes.
save-metadata Manually write node metadata to disk without running
destroy Delete Ursula node configuration.
"""
#
# Boring Setup Stuff
#
log = Logger('ursula.cli')
if debug:
click_config.log_to_sentry = False
click_config.log_to_file = True
globalLogPublisher.removeObserver(logToSentry) # Sentry
globalLogPublisher.addObserver(SimpleObserver(log_level_name='debug')) # Print
#
# Launch Warnings
#
if dev:
click.secho("WARNING: Running in development mode", fg='yellow')
if federated_only:
click.secho("WARNING: Running in Federated mode", fg='yellow')
if force:
click.secho("WARNING: Force is enabled", fg='yellow')
#
# Unauthenticated Configurations
#
if action == "init":
"""Create a brand-new persistent Ursula"""
if dev:
click.secho("WARNING: Using temporary storage area", fg='yellow')
if not config_root: # Flag
config_root = click_config.config_file # Envvar
if not rest_host:
rest_host = click.prompt("Enter Ursula's public-facing IPv4 address")
ursula_config = UrsulaConfiguration.generate(password=click_config.get_password(confirm=True),
config_root=config_root,
rest_host=rest_host,
rest_port=rest_port,
db_filepath=db_filepath,
federated_only=federated_only,
checksum_public_address=checksum_address,
no_registry=federated_only or no_registry,
registry_filepath=registry_filepath,
provider_uri=provider_uri)
click.secho("Generated keyring {}".format(ursula_config.keyring_dir), fg='green')
click.secho("Saved configuration file {}".format(ursula_config.config_file_location), fg='green')
# Give the use a suggestion as to what to do next...
how_to_run_message = "\nTo run an Ursula node from the default configuration filepath run: \n\n'{}'\n"
suggested_command = 'nucypher ursula run'
if config_root is not None:
config_file_location = os.path.join(config_root, config_file or UrsulaConfiguration.CONFIG_FILENAME)
suggested_command += ' --config-file {}'.format(config_file_location)
click.secho(how_to_run_message.format(suggested_command), fg='green')
return # FIN
# Development Configuration
if dev:
ursula_config = UrsulaConfiguration(dev_mode=True,
poa=poa,
registry_filepath=registry_filepath,
provider_uri=provider_uri,
checksum_public_address=checksum_address,
federated_only=federated_only,
rest_host=rest_host,
rest_port=rest_port,
db_filepath=db_filepath)
# Authenticated Configurations
else:
# Restore configuration from file
ursula_config = UrsulaConfiguration.from_configuration_file(filepath=config_file
# TODO: CLI Overrides for file-based configurations
# poa = poa,
# registry_filepath = registry_filepath,
# provider_uri = provider_uri,
# checksum_public_address = checksum_public_address,
# federated_only = federated_only,
# rest_host = rest_host,
# rest_port = rest_port,
# db_filepath = db_filepath
)
try: # Unlock Keyring
# ursula_config.attach_keyring()
click.secho('Decrypting keyring...', fg='blue')
ursula_config.keyring.unlock(password=click_config.get_password()) # Takes ~3 seconds, ~1GB Ram
except CryptoError:
raise ursula_config.keyring.AuthenticationFailed
click_config.ursula_config = ursula_config # Pass Ursula's config onto staking sub-command
#
# Action Switch
#
if action == 'run':
"""Seed, Produce, Run!"""
#
# Seed - Step 1
#
teacher_nodes = list()
if teacher_uri:
node = Ursula.from_teacher_uri(teacher_uri=teacher_uri, min_stake=min_stake, federated_only=federated_only)
teacher_nodes.append(node)
#
# Produce - Step 2
#
ursula = ursula_config.produce(known_nodes=teacher_nodes)
ursula_config.log.debug("Initialized Ursula {}".format(ursula), fg='green')
# GO!
try:
#
# Run - Step 3
#
click.secho("Running Ursula on {}".format(ursula.rest_interface), fg='green', bold=True)
if not debug:
stdio.StandardIO(UrsulaCommandProtocol(ursula=ursula))
ursula.get_deployer().run()
except Exception as e:
ursula_config.log.critical(str(e))
click.secho("{} {}".format(e.__class__.__name__, str(e)), fg='red')
raise # Crash :-(
finally:
click.secho("Stopping Ursula")
ursula_config.cleanup()
click.secho("Ursula Stopped", fg='red')
return
elif action == "save-metadata":
"""Manually save a node self-metadata file"""
ursula = ursula_config.produce(ursula_config=ursula_config)
metadata_path = ursula.write_node_metadata(node=ursula)
click.secho("Successfully saved node metadata to {}.".format(metadata_path), fg='green')
return
elif action == "view":
"""Paint an existing configuration to the console"""
paint_configuration(config_filepath=config_file or ursula_config.config_file_location)
return
elif action == "forget":
"""Forget all known nodes via storages"""
click.confirm("Permanently delete all known node data?", abort=True)
ursula_config.forget_nodes()
message = "Removed all stored node node metadata and certificates"
click.secho(message=message, fg='red')
return
elif action == "destroy":
"""Delete all configuration files from the disk"""
if not force:
click.confirm('''
*Permanently and irreversibly delete all* nucypher files including:
- Private and Public Keys
- Known Nodes
- TLS certificates
- Node Configurations
- Log Files
Delete {}?'''.format(ursula_config.config_root), abort=True)
try:
ursula_config.destroy(force=force)
except FileNotFoundError:
message = 'Failed: No nucypher files found at {}'.format(ursula_config.config_root)
click.secho(message, fg='red')
log.debug(message)
raise click.Abort()
else:
message = "Deleted configuration files at {}".format(ursula_config.config_root)
click.secho(message, fg='green')
log.debug(message)
return
else:
raise click.BadArgumentUsage("No such argument {}".format(action))
@click.argument('action', default='list', required=False)
@click.option('--checksum-address', type=EIP55_CHECKSUM_ADDRESS)
@click.option('--value', help="Token value of stake", type=STAKE_VALUE)
@click.option('--duration', help="Period duration of stake", type=STAKE_DURATION)
@click.option('--index', help="A specific stake index to resume", type=click.INT)
@nucypher_click_config
def stake(click_config,
action,
checksum_address,
index,
value,
duration):
"""
Manage token staking. TODO
\b
Actions
-------------------------------------------------
\b
list List all stakes for this node.
init Stage a new stake.
confirm-activity Manually confirm-activity for the current period.
divide Divide an existing stake.
collect-reward Withdraw staking reward.
"""
ursula_config = click_config.ursula_config
#
# Initialize
#
if not ursula_config.federated_only:
ursula_config.connect_to_blockchain(click_config)
ursula_config.connect_to_contracts(click_config)
if not checksum_address:
if click_config.accounts == NO_BLOCKCHAIN_CONNECTION:
click.echo('No account found.')
raise click.Abort()
for index, address in enumerate(click_config.accounts):
if index == 0:
row = 'etherbase (0) | {}'.format(address)
else:
row = '{} .......... | {}'.format(index, address)
click.echo(row)
click.echo("Select ethereum address")
account_selection = click.prompt("Enter 0-{}".format(len(ur.accounts)), type=click.INT)
address = click_config.accounts[account_selection]
if action == 'list':
live_stakes = ursula_config.miner_agent.get_all_stakes(miner_address=checksum_address)
for index, stake_info in enumerate(live_stakes):
row = '{} | {}'.format(index, stake_info)
click.echo(row)
elif action == 'init':
click.confirm("Stage a new stake?", abort=True)
live_stakes = ursula_config.miner_agent.get_all_stakes(miner_address=checksum_address)
if len(live_stakes) > 0:
raise RuntimeError("There is an existing stake for {}".format(checksum_address))
# Value
balance = ursula_config.miner_agent.token_agent.get_balance(address=checksum_address)
click.echo("Current balance: {}".format(balance))
value = click.prompt("Enter stake value", type=click.INT)
# Duration
message = "Minimum duration: {} | Maximum Duration: {}".format(MIN_LOCKED_PERIODS, MAX_MINTING_PERIODS)
click.echo(message)
duration = click.prompt("Enter stake duration in periods (1 Period = 24 Hours)", type=click.INT)
start_period = ursula_config.miner_agent.get_current_period()
end_period = start_period + duration
# Review
click.echo("""
| Staged Stake |
Node: {address}
Value: {value}
Duration: {duration}
Start Period: {start_period}
End Period: {end_period}
""".format(address=checksum_address,
value=value,
duration=duration,
start_period=start_period,
end_period=end_period))
raise NotImplementedError
elif action == 'confirm-activity':
"""Manually confirm activity for the active period"""
stakes = ursula_config.miner_agent.get_all_stakes(miner_address=checksum_address)
if len(stakes) == 0:
raise RuntimeError("There are no active stakes for {}".format(checksum_address))
ursula_config.miner_agent.confirm_activity(node_address=checksum_address)
elif action == 'divide':
"""Divide an existing stake by specifying the new target value and end period"""
stakes = ursula_config.miner_agent.get_all_stakes(miner_address=checksum_address)
if len(stakes) == 0:
raise RuntimeError("There are no active stakes for {}".format(checksum_address))
if not index:
for selection_index, stake_info in enumerate(stakes):
click.echo("{} ....... {}".format(selection_index, stake_info))
index = click.prompt("Select a stake to divide", type=click.INT)
target_value = click.prompt("Enter new target value", type=click.INT)
extension = click.prompt("Enter number of periods to extend", type=click.INT)
click.echo("""
Current Stake: {}
New target value {}
New end period: {}
""".format(stakes[index],
target_value,
target_value + extension))
click.confirm("Is this correct?", abort=True)
ursula_config.miner_agent.divide_stake(miner_address=checksum_address,
stake_index=index,
value=value,
periods=extension)
elif action == 'collect-reward': # TODO: Implement
"""Withdraw staking reward to the specified wallet address"""
# click.confirm("Send {} to {}?".format)
# ursula_config.miner_agent.collect_staking_reward(collector_address=address)
raise NotImplementedError
else:
raise click.BadArgumentUsage("No such argument {}".format(action))

184
nucypher/cli/painting.py Normal file
View File

@ -0,0 +1,184 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import click
import maya
import nucypher
from constant_sorrow.constants import NO_KNOWN_NODES
from nucypher.config.characters import UrsulaConfiguration
from nucypher.config.constants import SEEDNODES
#
# Art
#
BANNER = """
_
| |
_ __ _ _ ___ _ _ _ __ | |__ ___ _ __
| '_ \| | | |/ __| | | | '_ \| '_ \ / _ \ '__|
| | | | |_| | (__| |_| | |_) | | | | __/ |
|_| |_|\__,_|\___|\__, | .__/|_| |_|\___|_|
__/ | |
|___/|_|
version {}
""".format(nucypher.__version__)
#
# Paint
#
def build_fleet_state_status(ursula) -> str:
# Build FleetState status line
if ursula.known_nodes.checksum is not NO_KNOWN_NODES:
fleet_state_checksum = ursula.known_nodes.checksum[:7]
fleet_state_nickname = ursula.known_nodes.nickname
fleet_state_icon = ursula.known_nodes.icon
fleet_state = '{checksum} ⇀{nickname}↽ {icon}'.format(icon=fleet_state_icon,
nickname=fleet_state_nickname,
checksum=fleet_state_checksum)
elif ursula.known_nodes.checksum is not NO_KNOWN_NODES:
fleet_state = 'No Known Nodes'
else:
fleet_state = 'Unknown'
return fleet_state
def paint_configuration(config_filepath: str) -> None:
json_config = UrsulaConfiguration._read_configuration_file(filepath=config_filepath)
click.secho("\n======== Ursula Configuration ======== \n", bold=True)
for key, value in json_config.items():
click.secho("{} = {}".format(key, value))
def paint_node_status(ursula, start_time):
# Build Learning status line
learning_status = "Unknown"
if ursula._learning_task.running:
learning_status = "Learning at {}s Intervals".format(ursula._learning_task.interval)
elif not ursula._learning_task.running:
learning_status = "Not Learning"
teacher = 'Current Teacher ..... No Teacher Connection'
if ursula._current_teacher_node:
teacher = 'Current Teacher ..... {}'.format(ursula._current_teacher_node)
# Build FleetState status line
fleet_state = build_fleet_state_status(ursula=ursula)
stats = ['⇀URSULA {}↽'.format(ursula.nickname_icon),
'{}'.format(ursula),
'Uptime .............. {}'.format(maya.now() - start_time),
'Start Time .......... {}'.format(start_time.slang_time()),
'Fleet State.......... {}'.format(fleet_state),
'Learning Status ..... {}'.format(learning_status),
'Learning Round ...... Round #{}'.format(ursula._learning_round),
'Operating Mode ...... {}'.format('Federated' if ursula.federated_only else 'Decentralized'),
'Rest Interface ...... {}'.format(ursula.rest_url()),
'Node Storage Type ... {}'.format(ursula.node_storage._name.capitalize()),
'Known Nodes ......... {}'.format(len(ursula.known_nodes)),
'Work Orders ......... {}'.format(len(ursula._work_orders)),
teacher]
click.echo('\n' + '\n'.join(stats) + '\n')
def paint_known_nodes(ursula) -> None:
# Gather Data
known_nodes = ursula.known_nodes
number_of_known_nodes = len(ursula.node_storage.all(federated_only=ursula.federated_only))
seen_nodes = len(ursula.node_storage.all(federated_only=ursula.federated_only, certificates_only=True))
# Operating Mode
federated_only = ursula.federated_only
if federated_only:
click.secho("Configured in Federated Only mode", fg='green')
# Heading
label = "Known Nodes (connected {} / seen {})".format(number_of_known_nodes, seen_nodes)
heading = '\n' + label + " " * (45 - len(label))
click.secho(heading, bold=True, nl=True)
# Build FleetState status line
fleet_state = build_fleet_state_status(ursula=ursula)
fleet_status_line = 'Fleet State {}'.format(fleet_state)
click.secho(fleet_status_line, fg='blue', bold=True, nl=True)
# Legend
color_index = {
'self': 'yellow',
'known': 'white',
'seednode': 'blue'
}
# Ledgend
# for node_type, color in color_index.items():
# click.secho('{0:<6} | '.format(node_type), fg=color, nl=False)
# click.echo('\n')
seednode_addresses = list(bn.checksum_address for bn in SEEDNODES)
for node in known_nodes:
row_template = "{} | {}"
node_type = 'known'
if node.checksum_public_address == ursula.checksum_public_address:
node_type = 'self'
row_template += ' ({})'.format(node_type)
elif node.checksum_public_address in seednode_addresses:
node_type = 'seednode'
row_template += ' ({})'.format(node_type)
click.secho(row_template.format(node.rest_url().ljust(20), node), fg=color_index[node_type])
def paint_contract_status(ursula_config, click_config):
contract_payload = """
| NuCypher ETH Contracts |
Provider URI ............. {provider_uri}
Registry Path ............ {registry_filepath}
NucypherToken ............ {token}
MinerEscrow .............. {escrow}
PolicyManager ............ {manager}
""".format(provider_uri=ursula_config.blockchain.interface.provider_uri,
registry_filepath=ursula_config.blockchain.interface.registry.filepath,
token=ursula_config.token_agent.contract_address,
escrow=ursula_config.miner_agent.contract_address,
manager=ursula_config.policy_agent.contract_address,
period=ursula_config.miner_agent.get_current_period())
click.secho(contract_payload)
network_payload = """
| Blockchain Network |
Current Period ........... {period}
Gas Price ................ {gas_price}
Active Staking Ursulas ... {ursulas}
""".format(period=click_config.miner_agent.get_current_period(),
gas_price=click_config.blockchain.interface.w3.eth.gasPrice,
ursulas=click_config.miner_agent.get_miner_population())
click.secho(network_payload)

110
nucypher/cli/protocol.py Normal file
View File

@ -0,0 +1,110 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
from collections import deque
import click
import maya
from twisted.internet import reactor
from twisted.protocols.basic import LineReceiver
from nucypher.cli.painting import build_fleet_state_status
class UrsulaCommandProtocol(LineReceiver):
encoding = 'utf-8'
delimiter = os.linesep.encode(encoding=encoding)
def __init__(self, ursula):
self.ursula = ursula
self.start_time = maya.now()
self.__history = deque(maxlen=10)
self.prompt = bytes('Ursula({}) >>> '.format(self.ursula.checksum_public_address[:9]), encoding='utf-8')
# Expose Ursula functional entry points
self.__commands = {
# Status
'status': self.paintStatus,
'known_nodes': self.paintKnownNodes,
'fleet_state': self.paintFleetState,
# Learning Control
'cycle_teacher': self.ursula.cycle_teacher_node,
'start_learning': self.ursula.start_learning_loop,
'stop_learning': self.ursula.stop_learning_loop,
# Process Control
'stop': reactor.stop,
}
super().__init__()
@property
def commands(self):
return self.__commands.keys()
def paintKnownNodes(self):
from nucypher.cli.painting import paint_known_nodes
paint_known_nodes(ursula=self.ursula)
def paintStatus(self):
from nucypher.cli.painting import paint_node_status
paint_node_status(ursula=self.ursula, start_time=self.start_time)
def paintFleetState(self):
line = '{}'.format(build_fleet_state_status(ursula=self.ursula))
click.secho(line)
def connectionMade(self):
message = 'Attached {}@{}'.format(
self.ursula.checksum_public_address,
self.ursula.rest_url())
click.secho(message, fg='green')
click.secho('{} | {}'.format(self.ursula.nickname_icon, self.ursula.nickname), fg='blue', bold=True)
click.secho("\nType 'help' or '?' for help")
self.transport.write(self.prompt)
def lineReceived(self, line):
"""Ursula Console REPL"""
# Read
raw_line = line.decode(encoding=self.encoding)
line = raw_line.strip().lower()
# Evaluate
try:
self.__commands[line]()
# Print
except KeyError:
if line: # allow for empty string
click.secho("Invalid input. Options are {}".format(', '.join(self.__commands.keys())))
else:
self.__history.append(raw_line)
# Loop
self.transport.write(self.prompt)

54
nucypher/cli/types.py Normal file
View File

@ -0,0 +1,54 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from ipaddress import ip_address
import click
from eth_utils import is_checksum_address
from nucypher.blockchain.eth.constants import MIN_ALLOWED_LOCKED, MAX_MINTING_PERIODS, MIN_LOCKED_PERIODS, \
MAX_ALLOWED_LOCKED
class ChecksumAddress(click.ParamType):
name = 'checksum_public_address'
def convert(self, value, param, ctx):
if is_checksum_address(value):
return value
self.fail('{} is not a valid EIP-55 checksum address'.format(value, param, ctx))
class IPv4Address(click.ParamType):
name = 'ipv4_address'
def convert(self, value, param, ctx):
try:
_address = ip_address(value)
except ValueError as e:
self.fail(str(e))
else:
return value
STAKE_DURATION = click.IntRange(min=MIN_LOCKED_PERIODS, max=MAX_MINTING_PERIODS, clamp=False)
STAKE_VALUE = click.IntRange(min=MIN_ALLOWED_LOCKED, max=MAX_ALLOWED_LOCKED, clamp=False)
EXISTING_WRITABLE_DIRECTORY = click.Path(exists=True, dir_okay=True, file_okay=False, writable=True)
EXISTING_READABLE_FILE = click.Path(exists=True, dir_okay=False, file_okay=True, readable=True)
UNREGISTERED_PORT = click.IntRange(min=49151, max=65535, clamp=False)
IPV4_ADDRESS = IPv4Address()
EIP55_CHECKSUM_ADDRESS = ChecksumAddress()

148
nucypher/config/characters.py
View File

@ -14,134 +14,88 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
from constant_sorrow import constants
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurve
from cryptography.x509 import Certificate
from web3.middleware import geth_poa_middleware
from constant_sorrow.constants import (
UNINITIALIZED_CONFIGURATION,
NO_KEYRING_ATTACHED
)
from nucypher.blockchain.eth.agents import NucypherTokenAgent, MinerAgent
from nucypher.blockchain.eth.chains import Blockchain
from nucypher.config.constants import DEFAULT_CONFIG_ROOT
from nucypher.config.node import NodeConfiguration
from nucypher.crypto.powers import CryptoPower
class UrsulaConfiguration(NodeConfiguration):
from nucypher.characters.lawful import Ursula
_character_class = Ursula
_name = 'ursula'
DEFAULT_CONFIG_FILE_LOCATION = os.path.join(DEFAULT_CONFIG_ROOT, '{}.config'.format(_name))
DEFAULT_REST_HOST = '127.0.0.1'
DEFAULT_REST_PORT = 9151
_CHARACTER_CLASS = Ursula
_NAME = 'ursula'
__DB_TEMPLATE = "ursula.{port}.db"
DEFAULT_DB_NAME = __DB_TEMPLATE.format(port=DEFAULT_REST_PORT)
__DEFAULT_TLS_CURVE = ec.SECP384R1
CONFIG_FILENAME = '{}.config'.format(_NAME)
DEFAULT_CONFIG_FILE_LOCATION = os.path.join(DEFAULT_CONFIG_ROOT, CONFIG_FILENAME)
DEFAULT_DB_NAME = '{}.db'.format(_NAME)
def __init__(self,
rest_host: str = None,
rest_port: int = None,
# TLS
tls_curve: EllipticCurve = None,
certificate: Certificate = None,
certificate_filepath: str = None,
# Ursula
db_name: str = None,
dev_mode: bool = False,
db_filepath: str = None,
interface_signature=None,
crypto_power: CryptoPower = None,
# Blockchain
poa: bool = False,
provider_uri: str = None,
*args, **kwargs
) -> None:
# REST
self.rest_host = rest_host or self.DEFAULT_REST_HOST
self.rest_port = rest_port or self.DEFAULT_REST_PORT
self.db_name = db_name or self.__DB_TEMPLATE.format(port=self.rest_port)
self.db_filepath = db_filepath or constants.UNINITIALIZED_CONFIGURATION
#
# TLS
#
self.tls_curve = tls_curve or self.__DEFAULT_TLS_CURVE
self.certificate = certificate
self.certificate_filepath = certificate_filepath
# Ursula
self.interface_signature = interface_signature
self.crypto_power = crypto_power
#
# Blockchain
#
self.poa = poa
self.provider_uri = provider_uri
super().__init__(*args, **kwargs)
*args, **kwargs) -> None:
if dev_mode is True:
db_filepath = ':memory:' # sqlite in-memory db
self.db_filepath = db_filepath or UNINITIALIZED_CONFIGURATION
super().__init__(dev_mode=dev_mode, *args, **kwargs)
def generate_runtime_filepaths(self, config_root: str) -> dict:
base_filepaths = NodeConfiguration.generate_runtime_filepaths(config_root=config_root)
filepaths = dict(db_filepath=os.path.join(config_root, self.db_name))
base_filepaths = super().generate_runtime_filepaths(config_root=config_root)
filepaths = dict(db_filepath=os.path.join(config_root, self.DEFAULT_DB_NAME))
base_filepaths.update(filepaths)
return base_filepaths
def initialize(self, tls: bool = True, host=None, *args, **kwargs):
super().initialize(tls=tls, host=host or self.rest_host, curve=self.tls_curve, *args, **kwargs)
if self.db_name is constants.UNINITIALIZED_CONFIGURATION:
self.db_name = self.__DB_TEMPLATE.format(self.rest_port)
if self.db_filepath is constants.UNINITIALIZED_CONFIGURATION:
self.db_filepath = os.path.join(self.config_root, self.db_name)
@property
def static_payload(self) -> dict:
payload = dict(
rest_host=self.rest_host,
rest_port=self.rest_port,
db_name=self.db_name,
db_filepath=self.db_filepath,
)
if not self.temp:
certificate_filepath = self.certificate_filepath or self.keyring.certificate_filepath
payload.update(dict(certificate_filepath=certificate_filepath))
return {**super().static_payload, **payload}
@property
def dynamic_payload(self) -> dict:
payload = dict(
network_middleware=self.network_middleware,
tls_curve=self.tls_curve, # TODO: Needs to be in static payload with mapping
tls_curve=self.tls_curve, # TODO: Needs to be in static payload with [str -> curve] mapping
certificate=self.certificate,
interface_signature=self.interface_signature,
timestamp=None,
)
return {**super().dynamic_payload, **payload}
def produce(self, passphrase: str = None, **overrides):
def produce(self, **overrides):
"""Produce a new Ursula from configuration"""
if not self.temp:
self.read_keyring()
self.keyring.unlock(passphrase=passphrase)
# Build a merged dict of Ursula parameters
merged_parameters = {**self.static_payload, **self.dynamic_payload, **overrides}
#
# Pre-Init
#
# Verify the configuration file refers to the same configuration root as this instance
config_root_from_config_file = merged_parameters.pop('config_root')
if config_root_from_config_file != self.config_root:
message = "Configuration root mismatch {} and {}.".format(config_root_from_config_file, self.config_root)
raise self.ConfigurationError(message)
if self.federated_only is False:
self.blockchain = Blockchain.connect(provider_uri=self.provider_uri)
if self.poa: # TODO: move this..?
if self.poa:
w3 = self.miner_agent.blockchain.interface.w3
w3.middleware_stack.inject(geth_poa_middleware, layer=0)
@ -149,9 +103,15 @@ class UrsulaConfiguration(NodeConfiguration):
self.miner_agent = MinerAgent(blockchain=self.blockchain)
merged_parameters.update(blockchain=self.blockchain)
ursula = self._character_class(**merged_parameters)
#
# Init
#
ursula = self._CHARACTER_CLASS(**merged_parameters)
if self.temp: # TODO: Move this..?
#
# Post-Init
#
if self.dev_mode:
class MockDatastoreThreadPool(object):
def callInThread(self, f, *args, **kwargs):
return f(*args, **kwargs)
@ -159,15 +119,33 @@ class UrsulaConfiguration(NodeConfiguration):
return ursula
def __write(self, password: str, no_registry: bool):
_new_installation_path = self.initialize(password=password, import_registry=no_registry)
_configuration_filepath = self.to_configuration_file(filepath=self.config_file_location)
@classmethod
def generate(cls, password: str, no_registry: bool, *args, **kwargs) -> 'UrsulaConfiguration':
"""Hook-up a new initial installation and write configuration file to the disk"""
ursula_config = cls(dev_mode=False, is_me=True, *args, **kwargs)
ursula_config.__write(password=password, no_registry=no_registry)
return ursula_config
class AliceConfiguration(NodeConfiguration):
from nucypher.characters.lawful import Alice
_character_class = Alice
_name = 'alice'
_CHARACTER_CLASS = Alice
_NAME = 'alice'
CONFIG_FILENAME = '{}.config'.format(_NAME)
DEFAULT_CONFIG_FILE_LOCATION = os.path.join(DEFAULT_CONFIG_ROOT, CONFIG_FILENAME)
class BobConfiguration(NodeConfiguration):
from nucypher.characters.lawful import Bob
_character_class = Bob
_name = 'bob'
_CHARACTER_CLASS = Bob
_NAME = 'bob'
CONFIG_FILENAME = '{}.config'.format(_NAME)
DEFAULT_CONFIG_FILE_LOCATION = os.path.join(DEFAULT_CONFIG_ROOT, CONFIG_FILENAME)

2
nucypher/config/constants.py
View File

@ -34,5 +34,5 @@ DEFAULT_CONFIG_ROOT = APP_DIR.user_data_dir
USER_LOG_DIR = APP_DIR.user_log_dir
# Static Seednodes
SeednodeMetadata = namedtuple('seednode', ['checksum_address', 'rest_host', 'rest_port'])
SeednodeMetadata = namedtuple('seednode', ['checksum_public_address', 'rest_host', 'rest_port'])
SEEDNODES = tuple()

90
nucypher/config/keyring.py
View File

@ -19,6 +19,8 @@ import json
import os
import stat
from json import JSONDecodeError
from cryptography.hazmat.primitives.asymmetric import ec
from typing import ClassVar, Tuple, Callable, Union, Dict
from constant_sorrow import constants
@ -188,9 +190,7 @@ def _read_tls_public_certificate(filepath: str) -> Certificate:
# Encrypt and Decrypt
#
def _derive_key_material_from_passphrase(salt: bytes,
passphrase: str
) -> bytes:
def _derive_key_material_from_password(salt: bytes, password: str) -> bytes:
"""
Uses Scrypt derivation to derive a key for encrypting key material.
See RFC 7914 for n, r, and p value selections.
@ -204,7 +204,7 @@ def _derive_key_material_from_passphrase(salt: bytes,
r=8,
p=1,
backend=default_backend()
).derive(passphrase.encode())
).derive(password.encode())
except InternalError as e:
# OpenSSL Attempts to malloc 1 GB of mem for scrypt key derivation
if e.err_code[0].reason == 65:
@ -278,10 +278,10 @@ def _generate_signing_keys() -> Tuple[UmbralPrivateKey, UmbralPublicKey]:
return privkey, pubkey
def _generate_wallet(passphrase: str) -> Tuple[str, dict]:
"""Create a new wallet address and private "transacting" key encrypted with the passphrase"""
def _generate_wallet(password: str) -> Tuple[str, dict]:
"""Create a new wallet address and private "transacting" key encrypted with the password"""
account = Account.create(extra_entropy=os.urandom(32)) # max out entropy for keccak256
encrypted_wallet_data = Account.encrypt(private_key=account.privateKey, password=passphrase)
encrypted_wallet_data = Account.encrypt(private_key=account.privateKey, password=password)
return account.address, encrypted_wallet_data
@ -358,6 +358,7 @@ class NucypherKeyring:
__default_keyring_root = os.path.join(DEFAULT_CONFIG_ROOT, 'keyring')
_private_key_serializer = _PrivateKeySerializer()
__DEFAULT_TLS_CURVE = ec.SECP384R1
class KeyringError(Exception):
pass
@ -365,7 +366,7 @@ class NucypherKeyring:
class KeyringLocked(KeyringError):
pass
class InvalidPassphrase(KeyringError):
class AuthenticationFailed(KeyringError):
pass
def __init__(self,
@ -422,7 +423,7 @@ class NucypherKeyring:
@property
def checksum_address(self) -> str:
key_data = _read_keyfile(keypath=self.__wallet_path, deserializer=None)
# TODO Json joads
# TODO Json joads # TODO: what is this TODO?
address = key_data['address']
return to_checksum_address(address)
@ -477,12 +478,12 @@ class NucypherKeyring:
return __key_filepaths
def _export_wallet_to_node(self, blockchain, passphrase): # TODO: Deprecate with geth.parity signing EIPs
"""Decrypt the wallet with a passphrase, then import the key to the nodes's keyring over RPC"""
def _export_wallet_to_node(self, blockchain, password): # TODO: Deprecate with geth.parity signing EIPs
"""Decrypt the wallet with a password, then import the key to the nodes's keyring over RPC"""
with open(self.__wallet_path, 'rb') as wallet:
data = wallet.read().decode(FILE_ENCODING)
account = Account.decrypt(keyfile_json=data, password=passphrase)
blockchain.interface.w3.personal.importRawKey(private_key=account, passphrase=passphrase)
account = Account.decrypt(keyfile_json=data, password=password)
blockchain.interface.w3.personal.importRawKey(private_key=account, password=password)
@unlock_required
def __decrypt_keyfile(self, key_path: str) -> UmbralPrivateKey:
@ -510,12 +511,12 @@ class NucypherKeyring:
self.__derived_key_material = constants.KEYRING_LOCKED
return self.is_unlocked
def unlock(self, passphrase: str) -> bool:
def unlock(self, password: str) -> bool:
if self.is_unlocked:
return self.is_unlocked
key_data = _read_keyfile(keypath=self.__root_keypath, deserializer=self._private_key_serializer)
try:
derived_key = _derive_key_material_from_passphrase(passphrase=passphrase, salt=key_data['master_salt'])
derived_key = _derive_key_material_from_password(password=password, salt=key_data['master_salt'])
except CryptoError:
raise
else:
@ -568,7 +569,7 @@ class NucypherKeyring:
#
@classmethod
def generate(cls,
passphrase: str,
password: str,
encrypting: bool = True,
wallet: bool = True,
tls: bool = True,
@ -577,28 +578,28 @@ class NucypherKeyring:
keyring_root: str = None,
) -> 'NucypherKeyring':
"""
Generates new encrypting, signing, and wallet keys encrypted with the passphrase,
Generates new encrypting, signing, and wallet keys encrypted with the password,
respectively saving keyfiles on the local filesystem from *default* paths,
returning the corresponding Keyring instance.
"""
failures = cls.validate_passphrase(passphrase)
failures = cls.validate_password(password)
if failures:
raise cls.InvalidPassphrase(", ".join(failures)) # TODO: Ensure this scope is seperable from the scope containing the passphrase
raise cls.AuthenticationFailed(", ".join(failures)) # TODO: Ensure this scope is seperable from the scope containing the password
if not any((wallet, encrypting, tls)):
raise ValueError('Either "encrypting", "wallet", or "tls" must be True '
'to generate new keys, or set "no_keys" to True to skip generation.')
if curve is None:
curve = cls.__DEFAULT_TLS_CURVE
_base_filepaths = cls._generate_base_filepaths(keyring_root=keyring_root)
_public_key_dir = _base_filepaths['public_key_dir']
_private_key_dir = _base_filepaths['private_key_dir']
# Create the key directories with default paths. Raises OSError if dirs exist
# if exists_ok and not os.path.isdir(_public_key_dir):
# Write to disk
os.mkdir(_public_key_dir, mode=0o744) # public dir
# if exists_ok and not os.path.isdir(_private_key_dir):
os.mkdir(_private_key_dir, mode=0o700) # private dir
#
@ -608,10 +609,11 @@ class NucypherKeyring:
keyring_args = dict()
if wallet is True:
new_address, new_wallet = _generate_wallet(passphrase)
new_address, new_wallet = _generate_wallet(password)
new_wallet_path = os.path.join(_private_key_dir, 'wallet-{}.json'.format(new_address))
saved_wallet_path = _write_private_keyfile(new_wallet_path, json.dumps(new_wallet), serializer=None)
keyring_args.update(wallet_path=saved_wallet_path)
with open(new_wallet_path, 'w') as wallet: # TODO: is this pub or private?
wallet.write(json.dumps(new_wallet))
keyring_args.update(wallet_path=new_wallet_path)
account = new_address
if encrypting is True:
@ -630,33 +632,27 @@ class NucypherKeyring:
delegating_keying_material = UmbralKeyingMaterial().to_bytes()
# Derive Wrapping Keys
passphrase_salt, encrypting_salt, signing_salt, delegating_salt = (os.urandom(32) for _ in range(4))
derived_key_material = _derive_key_material_from_passphrase(salt=passphrase_salt,
passphrase=passphrase)
encrypting_wrap_key = _derive_wrapping_key_from_key_material(salt=encrypting_salt,
key_material=derived_key_material)
signature_wrap_key = _derive_wrapping_key_from_key_material(salt=signing_salt,
key_material=derived_key_material)
delegating_wrap_key = _derive_wrapping_key_from_key_material(salt=delegating_salt,
key_material=derived_key_material)
password_salt, encrypting_salt, signing_salt, delegating_salt = (os.urandom(32) for _ in range(4))
derived_key_material = _derive_key_material_from_password(salt=password_salt, password=password)
encrypting_wrap_key = _derive_wrapping_key_from_key_material(salt=encrypting_salt, key_material=derived_key_material)
signature_wrap_key = _derive_wrapping_key_from_key_material(salt=signing_salt, key_material=derived_key_material)
delegating_wrap_key = _derive_wrapping_key_from_key_material(salt=delegating_salt, key_material=derived_key_material)
# TODO: Deprecate _encrypt_umbral_key with new pyumbral release
# Encapsulate Private Keys
encrypting_key_data = _encrypt_umbral_key(umbral_key=encrypting_private_key,
wrapping_key=encrypting_wrap_key)
signing_key_data = _encrypt_umbral_key(umbral_key=signing_private_key,
wrapping_key=signature_wrap_key)
encrypting_key_data = _encrypt_umbral_key(umbral_key=encrypting_private_key, wrapping_key=encrypting_wrap_key)
signing_key_data = _encrypt_umbral_key(umbral_key=signing_private_key, wrapping_key=signature_wrap_key)
delegating_key_data = bytes(SecretBox(delegating_wrap_key).encrypt(delegating_keying_material))
# Assemble Private Keys
encrypting_key_metadata = _assemble_key_data(key_data=encrypting_key_data,
master_salt=passphrase_salt,
master_salt=password_salt,
wrap_salt=encrypting_salt)
signing_key_metadata = _assemble_key_data(key_data=signing_key_data,
master_salt=passphrase_salt,
master_salt=password_salt,
wrap_salt=signing_salt)
delegating_key_metadata = _assemble_key_data(key_data=delegating_key_data,
master_salt=passphrase_salt,
master_salt=password_salt,
wrap_salt=delegating_salt)
# Write Private Keys
@ -686,7 +682,7 @@ class NucypherKeyring:
if tls is True:
if not all((host, curve)):
raise ValueError("Host and curve are required to make a new keyring TLS certificate")
raise ValueError("Host and curve are required to make a new keyring TLS certificate. Got {}, {}".format(host, curve))
private_key, cert = _generate_tls_keys(host, curve)
def __serialize_pem(pk):
@ -704,15 +700,15 @@ class NucypherKeyring:
return keyring_instance
@staticmethod
def validate_passphrase(passphrase: str) -> bool:
def validate_password(password: str) -> bool:
"""
Validate a passphrase and return True or raise an error with a failure reason.
Validate a password and return True or raise an error with a failure reason.
NOTICE: Do not raise inside this function.
"""
rules = (
(bool(passphrase), 'Passphrase must not be blank.'),
(len(passphrase) >= 16, 'Passphrase is too short, must be >= 16 chars.'),
(bool(password), 'Password must not be blank.'),
(len(password) >= 16, 'Password is too short, must be >= 16 chars.'),
)
failures = list()

444
nucypher/config/node.py
View File

@ -19,41 +19,69 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
import binascii
import json
import os
import secrets
import string
from abc import ABC, abstractmethod
from json import JSONDecodeError
from tempfile import TemporaryDirectory
from typing import List
from constant_sorrow.constants import UNINITIALIZED_CONFIGURATION, STRANGER_CONFIGURATION, LIVE_CONFIGURATION
import shutil
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurve
from cryptography.x509 import Certificate
from twisted.logger import Logger
from typing import List
from web3.middleware import geth_poa_middleware
from nucypher.characters.lawful import Ursula
from nucypher.config.constants import DEFAULT_CONFIG_ROOT, BASE_DIR
from constant_sorrow.constants import (
UNINITIALIZED_CONFIGURATION,
STRANGER_CONFIGURATION,
NO_BLOCKCHAIN_CONNECTION,
LIVE_CONFIGURATION,
NO_KEYRING_ATTACHED
)
from nucypher.blockchain.eth.agents import PolicyAgent, MinerAgent, NucypherTokenAgent
from nucypher.blockchain.eth.chains import Blockchain
from nucypher.config.constants import DEFAULT_CONFIG_ROOT, BASE_DIR, USER_LOG_DIR
from nucypher.config.keyring import NucypherKeyring
from nucypher.config.storages import NodeStorage, InMemoryNodeStorage, LocalFileBasedNodeStorage
from nucypher.crypto.powers import CryptoPowerUp
from nucypher.config.storages import NodeStorage, ForgetfulNodeStorage, LocalFileBasedNodeStorage
from nucypher.crypto.powers import CryptoPowerUp, CryptoPower
from nucypher.network.middleware import RestMiddleware
from nucypher.network.nodes import FleetStateTracker
from umbral.signing import Signature
class NodeConfiguration:
class NodeConfiguration(ABC):
"""
'Sideways Engagement' of Character classes; a reflection of input parameters.
"""
_name = 'ursula'
_character_class = Ursula
# Abstract
_NAME = NotImplemented
_CHARACTER_CLASS = NotImplemented
CONFIG_FILENAME = NotImplemented
DEFAULT_CONFIG_FILE_LOCATION = NotImplemented
DEFAULT_CONFIG_FILE_LOCATION = os.path.join(DEFAULT_CONFIG_ROOT, '{}.config'.format(_name))
# Mode
DEFAULT_OPERATING_MODE = 'decentralized'
NODE_SERIALIZER = binascii.hexlify
NODE_DESERIALIZER = binascii.unhexlify
# Configuration
__CONFIG_FILE_EXT = '.config'
__CONFIG_FILE_DESERIALIZER = json.loads
__TEMP_CONFIGURATION_DIR_PREFIX = "nucypher-tmp-"
__DEFAULT_NETWORK_MIDDLEWARE_CLASS = RestMiddleware
__DEFAULT_NODE_STORAGE = LocalFileBasedNodeStorage
# Registry
__REGISTRY_NAME = 'contract_registry.json'
REGISTRY_SOURCE = os.path.join(BASE_DIR, __REGISTRY_NAME) # TODO: #461 Where will this be hosted?
# Rest + TLS
DEFAULT_REST_HOST = '127.0.0.1'
DEFAULT_REST_PORT = 9151
__DEFAULT_TLS_CURVE = ec.SECP384R1
__DEFAULT_NETWORK_MIDDLEWARE_CLASS = RestMiddleware
class ConfigurationError(RuntimeError):
pass
@ -62,156 +90,273 @@ class NodeConfiguration:
def __init__(self,
temp: bool = False,
config_root: str = DEFAULT_CONFIG_ROOT,
# Base
config_root: str = None,
config_file_location: str = None,
passphrase: str = None,
auto_initialize: bool = False,
auto_generate_keys: bool = False,
config_file_location: str = DEFAULT_CONFIG_FILE_LOCATION,
keyring_dir: str = None,
checksum_address: str = None,
is_me: bool = True,
# Mode
dev_mode: bool = False,
federated_only: bool = False,
network_middleware: RestMiddleware = None,
registry_source: str = REGISTRY_SOURCE,
registry_filepath: str = None,
import_seed_registry: bool = False,
# Identity
is_me: bool = True,
checksum_public_address: str = None,
crypto_power: CryptoPower = None,
# Keyring
keyring: NucypherKeyring = None,
keyring_dir: str = None,
# Learner
learn_on_same_thread: bool = False,
abort_on_learning_error: bool = False,
start_learning_now: bool = True,
# TLS
known_certificates_dir: str = None,
# REST
rest_host: str = None,
rest_port: int = None,
# Metadata
# TLS
tls_curve: EllipticCurve = None,
certificate: Certificate = None,
# Network
interface_signature: Signature = None,
network_middleware: RestMiddleware = None,
# Node Storage
known_nodes: set = None,
node_storage: NodeStorage = None,
load_metadata: bool = True,
save_metadata: bool = True
reload_metadata: bool = True,
save_metadata: bool = True,
# Blockchain
poa: bool = False,
provider_uri: str = None,
# Registry
registry_source: str = None,
registry_filepath: str = None,
import_seed_registry: bool = False # TODO: needs cleanup
) -> None:
# Logs
self.log = Logger(self.__class__.__name__)
# Known Nodes
self.known_nodes_dir = UNINITIALIZED_CONFIGURATION
self.known_certificates_dir = known_certificates_dir or UNINITIALIZED_CONFIGURATION
#
# REST + TLS (Ursula)
#
self.rest_host = rest_host or self.DEFAULT_REST_HOST
self.rest_port = rest_port or self.DEFAULT_REST_PORT
self.tls_curve = tls_curve or self.__DEFAULT_TLS_CURVE
self.certificate = certificate
self.interface_signature = interface_signature
self.crypto_power = crypto_power
#
# Keyring
self.keyring = UNINITIALIZED_CONFIGURATION
#
self.keyring = keyring or NO_KEYRING_ATTACHED
self.keyring_dir = keyring_dir or UNINITIALIZED_CONFIGURATION
# Contract Registry
self.__registry_source = registry_source
if import_seed_registry is True:
registry_source = self.REGISTRY_SOURCE
if not os.path.isfile(registry_source):
message = "Seed contract registry does not exist at path {}.".format(registry_filepath)
self.log.debug(message)
raise RuntimeError(message)
self.__registry_source = registry_source or self.REGISTRY_SOURCE
self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION
# Configuration Root Directory
#
# Configuration
#
self.config_file_location = config_file_location or UNINITIALIZED_CONFIGURATION
self.config_root = UNINITIALIZED_CONFIGURATION
self.__temp = temp
if self.__temp:
self.__temp_dir = UNINITIALIZED_CONFIGURATION
self.node_storage = InMemoryNodeStorage(federated_only=federated_only,
character_class=self.__class__)
else:
self.config_root = config_root
self.__temp_dir = LIVE_CONFIGURATION
from nucypher.characters.lawful import Ursula # TODO : Needs cleanup
self.node_storage = node_storage or self.__DEFAULT_NODE_STORAGE(federated_only=federated_only,
character_class=Ursula)
self.__cache_runtime_filepaths()
self.config_file_location = config_file_location
#
# Mode
#
self.federated_only = federated_only
self.__dev_mode = dev_mode
if self.__dev_mode:
self.__temp_dir = UNINITIALIZED_CONFIGURATION
self.node_storage = ForgetfulNodeStorage(federated_only=federated_only, character_class=self.__class__)
else:
self.__temp_dir = LIVE_CONFIGURATION
self.config_root = config_root or DEFAULT_CONFIG_ROOT
self._cache_runtime_filepaths()
self.node_storage = node_storage or LocalFileBasedNodeStorage(federated_only=federated_only,
config_root=self.config_root)
#
# Identity
#
self.federated_only = federated_only
self.checksum_address = checksum_address
self.is_me = is_me
if self.is_me:
#
self.checksum_public_address = checksum_public_address
if self.is_me is True or dev_mode is True:
# Self
#
if checksum_address and not self.__temp:
self.read_keyring()
if self.checksum_public_address and dev_mode is False:
self.attach_keyring()
self.network_middleware = network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS()
else:
#
# Stranger
#
self.known_nodes_dir = STRANGER_CONFIGURATION
self.known_certificates_dir = STRANGER_CONFIGURATION
self.node_storage = STRANGER_CONFIGURATION
self.keyring_dir = STRANGER_CONFIGURATION
self.keyring = STRANGER_CONFIGURATION
self.network_middleware = STRANGER_CONFIGURATION
if network_middleware:
raise self.ConfigurationError("Cannot configure a stranger to use network middleware")
raise self.ConfigurationError("Cannot configure a stranger to use network middleware.")
#
# Learner
#
self.known_nodes = known_nodes or set()
self.learn_on_same_thread = learn_on_same_thread
self.abort_on_learning_error = abort_on_learning_error
self.start_learning_now = start_learning_now
self.save_metadata = save_metadata
self.load_metadata = load_metadata
self.reload_metadata = reload_metadata
self.__fleet_state = FleetStateTracker()
known_nodes = known_nodes or set()
if known_nodes:
self.known_nodes._nodes.update({node.checksum_public_address: node for node in known_nodes})
self.known_nodes.record_fleet_state()
#
# Auto-Initialization
# Blockchain
#
if auto_initialize:
self.initialize(no_registry=not import_seed_registry or federated_only,
wallet=auto_generate_keys and not federated_only,
encrypting=auto_generate_keys,
passphrase=passphrase)
self.poa = poa
self.provider_uri = provider_uri
self.blockchain = NO_BLOCKCHAIN_CONNECTION
self.accounts = NO_BLOCKCHAIN_CONNECTION
self.token_agent = NO_BLOCKCHAIN_CONNECTION
self.miner_agent = NO_BLOCKCHAIN_CONNECTION
self.policy_agent = NO_BLOCKCHAIN_CONNECTION
#
# Development Mode
#
if dev_mode:
# Ephemeral dev settings
self.abort_on_learning_error = True
self.save_metadata = False
self.reload_metadata = False
# Generate one-time alphanumeric development password
alphabet = string.ascii_letters + string.digits
password = ''.join(secrets.choice(alphabet) for _ in range(32))
# Auto-initialize
self.initialize(password=password, import_registry=import_seed_registry)
def __call__(self, *args, **kwargs):
return self.produce(*args, **kwargs)
def cleanup(self) -> None:
if self.__temp:
if self.__dev_mode:
self.__temp_dir.cleanup()
@property
def temp(self):
return self.__temp
def dev_mode(self):
return self.__dev_mode
def produce(self, passphrase: str = None, **overrides):
"""Initialize a new character instance and return it"""
if not self.temp:
self.read_keyring()
self.keyring.unlock(passphrase=passphrase)
@property
def known_nodes(self):
return self.__fleet_state
def connect_to_blockchain(self, provider_uri: str, poa: bool = False, compile_contracts: bool = False):
if self.federated_only:
raise NodeConfiguration.ConfigurationError("Cannot connect to blockchain in federated mode")
self.blockchain = Blockchain.connect(provider_uri=provider_uri, compile=compile_contracts)
if poa is True:
w3 = self.blockchain.interface.w3
w3.middleware_stack.inject(geth_poa_middleware, layer=0)
self.accounts = self.blockchain.interface.w3.eth.accounts
self.log.debug("Established connection to provider {}".format(self.blockchain.interface.provider_uri))
def connect_to_contracts(self) -> None:
"""Initialize contract agency and set them on config"""
self.token_agent = NucypherTokenAgent(blockchain=self.blockchain)
self.miner_agent = MinerAgent(blockchain=self.blockchain)
self.policy_agent = PolicyAgent(blockchain=self.blockchain)
self.log.debug("Established connection to nucypher contracts")
def read_known_nodes(self):
known_nodes = self.node_storage.all(federated_only=self.federated_only)
known_nodes = {node.checksum_public_address: node for node in known_nodes}
self.known_nodes._nodes.update(known_nodes)
self.known_nodes.record_fleet_state()
return self.known_nodes
def forget_nodes(self) -> None:
self.node_storage.clear()
message = "Removed all stored node node metadata and certificates"
self.log.debug(message)
def destroy(self, force: bool = False, logs: bool = True) -> None:
# TODO: Further confirm this is a nucypher dir first! (in-depth measure)
if logs is True or force:
shutil.rmtree(USER_LOG_DIR, ignore_errors=True)
try:
shutil.rmtree(self.config_root, ignore_errors=force)
except FileNotFoundError:
raise FileNotFoundError("No such directory {}".format(self.config_root))
def produce(self, **overrides):
"""Initialize a new character instance and return it."""
# Build a merged dict of node parameters
merged_parameters = {**self.static_payload, **self.dynamic_payload, **overrides}
return self._character_class(**merged_parameters)
# Verify the configuration file refers to the same configuration root as this instance
config_root_from_config_file = merged_parameters.pop('config_root')
if config_root_from_config_file != self.config_root:
message = "Configuration root mismatch {} and {}.".format(config_root_from_config_file, self.config_root)
raise self.ConfigurationError(message)
character = self._CHARACTER_CLASS(**merged_parameters)
return character
@staticmethod
def _read_configuration_file(filepath) -> dict:
with open(filepath, 'r') as file:
payload = NodeConfiguration.__CONFIG_FILE_DESERIALIZER(file.read())
def _read_configuration_file(filepath: str) -> dict:
try:
with open(filepath, 'r') as file:
raw_contents = file.read()
payload = NodeConfiguration.__CONFIG_FILE_DESERIALIZER(raw_contents)
except FileNotFoundError as e:
raise # TODO: Do we need better exception handling here?
return payload
@classmethod
def from_configuration_file(cls, filepath, **overrides) -> 'NodeConfiguration':
def from_configuration_file(cls, filepath: str = None, **overrides) -> 'NodeConfiguration':
"""Initialize a NodeConfiguration from a JSON file."""
from nucypher.config.storages import NodeStorage # TODO: move
NODE_STORAGES = {storage_class._name: storage_class for storage_class in NodeStorage.__subclasses__()}
from nucypher.config.storages import NodeStorage
node_storage_subclasses = {storage._name: storage for storage in NodeStorage.__subclasses__()}
if filepath is None:
filepath = cls.DEFAULT_CONFIG_FILE_LOCATION
# Read from disk
payload = cls._read_configuration_file(filepath=filepath)
# Make NodeStorage
# Initialize NodeStorage subclass from file (sub-configuration)
storage_payload = payload['node_storage']
storage_type = storage_payload[NodeStorage._TYPE_LABEL]
storage_class = NODE_STORAGES[storage_type]
storage_class = node_storage_subclasses[storage_type]
node_storage = storage_class.from_payload(payload=storage_payload,
character_class=cls._character_class,
character_class=cls._CHARACTER_CLASS,
federated_only=payload['federated_only'],
serializer=cls.NODE_SERIALIZER,
deserializer=cls.NODE_DESERIALIZER)
@ -222,11 +367,12 @@ class NodeConfiguration:
def to_configuration_file(self, filepath: str = None) -> str:
"""Write the static_payload to a JSON file."""
if filepath is None:
filename = '{}{}'.format(self._name.lower(), self.__CONFIG_FILE_EXT)
filename = '{}{}'.format(self._NAME.lower(), self.__CONFIG_FILE_EXT)
filepath = os.path.join(self.config_root, filename)
payload = self.static_payload
del payload['is_me'] # TODO
# Save node connection data
payload.update(dict(node_storage=self.node_storage.payload()))
@ -254,12 +400,13 @@ class NodeConfiguration:
def static_payload(self) -> dict:
"""Exported static configuration values for initializing Ursula"""
payload = dict(
config_root=self.config_root,
# Identity
is_me=self.is_me,
federated_only=self.federated_only, # TODO: 466
checksum_address=self.checksum_address,
checksum_public_address=self.checksum_public_address,
keyring_dir=self.keyring_dir,
known_certificates_dir=self.known_certificates_dir,
# Behavior
learn_on_same_thread=self.learn_on_same_thread,
@ -272,8 +419,12 @@ class NodeConfiguration:
@property
def dynamic_payload(self, **overrides) -> dict:
"""Exported dynamic configuration values for initializing Ursula"""
if self.load_metadata:
self.known_nodes.update(self.node_storage.all(federated_only=self.federated_only))
if self.reload_metadata:
known_nodes = self.node_storage.all(federated_only=self.federated_only)
known_nodes = {node.checksum_public_address: node for node in known_nodes}
self.known_nodes._nodes.update(known_nodes)
self.known_nodes.record_fleet_state()
payload = dict(network_middleware=self.network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS(),
known_nodes=self.known_nodes,
node_storage=self.node_storage,
@ -287,22 +438,19 @@ class NodeConfiguration:
def runtime_filepaths(self):
filepaths = dict(config_root=self.config_root,
keyring_dir=self.keyring_dir,
known_certificates_dir=self.known_certificates_dir,
registry_filepath=self.registry_filepath)
return filepaths
@staticmethod
def generate_runtime_filepaths(config_root: str) -> dict:
@classmethod
def generate_runtime_filepaths(cls, config_root: str) -> dict:
"""Dynamically generate paths based on configuration root directory"""
known_nodes_dir = os.path.join(config_root, 'known_nodes')
filepaths = dict(config_root=config_root,
config_file_location=os.path.join(config_root, cls.CONFIG_FILENAME),
keyring_dir=os.path.join(config_root, 'keyring'),
known_nodes_dir=known_nodes_dir,
known_certificates_dir=os.path.join(known_nodes_dir, 'certificates'),
registry_filepath=os.path.join(config_root, NodeConfiguration.__REGISTRY_NAME))
return filepaths
def __cache_runtime_filepaths(self) -> None:
def _cache_runtime_filepaths(self) -> None:
"""Generate runtime filepaths and cache them on the config object"""
filepaths = self.generate_runtime_filepaths(config_root=self.config_root)
for field, filepath in filepaths.items():
@ -311,28 +459,22 @@ class NodeConfiguration:
def derive_node_power_ups(self) -> List[CryptoPowerUp]:
power_ups = list()
if self.is_me and not self.temp:
for power_class in self._character_class._default_crypto_powerups:
if self.is_me and not self.dev_mode:
for power_class in self._CHARACTER_CLASS._default_crypto_powerups:
power_up = self.keyring.derive_crypto_power(power_class)
power_ups.append(power_up)
return power_ups
def initialize(self,
passphrase: str,
no_registry: bool = False,
wallet: bool = False,
encrypting: bool = False,
tls: bool = False,
host: str = None,
curve=None,
no_keys: bool = False
password: str,
import_registry: bool = True,
) -> str:
"""Write a new configuration to the disk, and with the configured node store."""
"""Initialize a new configuration."""
#
# Create Config Root
#
if self.__temp:
if self.__dev_mode:
self.__temp_dir = TemporaryDirectory(prefix=self.__TEMP_CONFIGURATION_DIR_PREFIX)
self.config_root = self.__temp_dir.name
else:
@ -348,62 +490,60 @@ class NodeConfiguration:
#
# Create Config Subdirectories
#
self.__cache_runtime_filepaths()
self._cache_runtime_filepaths()
try:
# Directories
os.mkdir(self.keyring_dir, mode=0o700) # keyring
os.mkdir(self.known_nodes_dir, mode=0o755) # known_nodes
os.mkdir(self.known_certificates_dir, mode=0o755) # known_certs
self.node_storage.initialize() # TODO: default known dir
# Node Storage
self.node_storage.initialize()
if not self.temp and not no_keys:
# Keyring
self.write_keyring(passphrase=passphrase,
wallet=wallet,
encrypting=encrypting,
tls=tls,
host=host,
tls_curve=curve)
# Keyring
if not self.dev_mode:
os.mkdir(self.keyring_dir, mode=0o700) # keyring TODO: Keyring backend entry point
self.write_keyring(password=password, host=self.rest_host, tls_curve=self.tls_curve)
# Registry
if not no_registry and not self.federated_only:
self.write_registry(output_filepath=self.registry_filepath,
source=self.__registry_source,
blank=no_registry)
if import_registry and not self.federated_only:
self.write_registry(output_filepath=self.registry_filepath, # type: str
source=self.__registry_source, # type: str
blank=import_registry) # type: bool
except FileExistsError:
existing_paths = [os.path.join(self.config_root, f) for f in os.listdir(self.config_root)]
message = "There are pre-existing nucypher installation files at {}: {}".format(self.config_root,
existing_paths)
message = "There are pre-existing files at {}: {}".format(self.config_root, existing_paths)
self.log.critical(message)
raise NodeConfiguration.ConfigurationError(message)
if not self.__temp:
self.validate(config_root=self.config_root, no_registry=no_registry or self.federated_only)
if not self.__dev_mode:
self.validate(config_root=self.config_root, no_registry=import_registry or self.federated_only)
# Success
message = "Created nucypher installation files at {}".format(self.config_root)
self.log.debug(message)
return self.config_root
def read_known_nodes(self):
self.known_nodes.update(self.node_storage.all(federated_only=self.federated_only))
return self.known_nodes
def attach_keyring(self, checksum_address: str = None, *args, **kwargs) -> None:
if self.keyring is not NO_KEYRING_ATTACHED:
if self.keyring.checksum_address != (checksum_address or self.checksum_public_address):
raise self.ConfigurationError("There is already a keyring attached to this configuration.")
return
def read_keyring(self, *args, **kwargs):
if self.checksum_address is None:
if (checksum_address or self.checksum_public_address) is None:
raise self.ConfigurationError("No account specified to unlock keyring")
self.keyring = NucypherKeyring(keyring_root=self.keyring_dir,
account=self.checksum_address,
self.keyring = NucypherKeyring(keyring_root=self.keyring_dir, # type: str
account=checksum_address or self.checksum_public_address, # type: str
*args, **kwargs)
def write_keyring(self,
passphrase: str,
encrypting: bool,
wallet: bool,
tls: bool,
host: str,
password: str,
encrypting: bool = True,
wallet: bool = False,
tls: bool = True,
tls_curve: EllipticCurve = None,
) -> NucypherKeyring:
self.keyring = NucypherKeyring.generate(passphrase=passphrase,
self.keyring = NucypherKeyring.generate(password=password,
encrypting=encrypting,
wallet=wallet,
tls=tls,
@ -413,11 +553,9 @@ class NodeConfiguration:
# TODO: Operating mode switch #466
if self.federated_only or not wallet:
self.checksum_address = self.keyring.federated_address
self.checksum_public_address = self.keyring.federated_address
else:
self.checksum_address = self.keyring.checksum_address
if tls:
self.certificate_filepath = self.keyring.certificate_filepath
self.checksum_public_address = self.keyring.checksum_address
return self.keyring
@ -434,7 +572,7 @@ class NodeConfiguration:
output_filepath = output_filepath or self.registry_filepath
source = source or self.REGISTRY_SOURCE
if not blank and not self.temp:
if not blank and not self.dev_mode:
# Validate Registry
with open(source, 'r') as registry_file:
try:
@ -450,5 +588,5 @@ class NodeConfiguration:
self.log.warn("Writing blank registry")
open(output_filepath, 'w').close() # write blank
self.log.info("Successfully wrote registry to {}".format(output_filepath))
self.log.debug("Successfully wrote registry to {}".format(output_filepath))
return output_filepath

485
nucypher/config/storages.py
View File

@ -14,19 +14,28 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import binascii
import glob
import os
import tempfile
from abc import abstractmethod, ABC
from twisted.logger import Logger
import OpenSSL
import boto3 as boto3
import shutil
from botocore.errorfactory import ClientError
from constant_sorrow import constants
from typing import Callable
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.x509 import Certificate
from twisted.logger import Logger
from typing import Callable, Tuple, Union, Set, Any
from constant_sorrow.constants import NO_STORAGE_AVAILIBLE
from nucypher.config.constants import DEFAULT_CONFIG_ROOT
from nucypher.utilities.decorators import validate_checksum_address
class NodeStorage(ABC):
@ -35,6 +44,8 @@ class NodeStorage(ABC):
_TYPE_LABEL = 'storage_type'
NODE_SERIALIZER = binascii.hexlify
NODE_DESERIALIZER = binascii.unhexlify
TLS_CERTIFICATE_ENCODING = Encoding.PEM
TLS_CERTIFICATE_EXTENSION = '.{}'.format(TLS_CERTIFICATE_ENCODING.name.lower())
class NodeStorageError(Exception):
pass
@ -43,23 +54,25 @@ class NodeStorage(ABC):
pass
def __init__(self,
character_class,
federated_only: bool, # TODO# 466
character_class=None,
serializer: Callable = NODE_SERIALIZER,
deserializer: Callable = NODE_DESERIALIZER,
) -> None:
from nucypher.characters.lawful import Ursula
self.log = Logger(self.__class__.__name__)
self.serializer = serializer
self.deserializer = deserializer
self.federated_only = federated_only
self.character_class = character_class
self.character_class = character_class or Ursula
def __getitem__(self, item):
return self.get(checksum_address=item, federated_only=self.federated_only)
def __setitem__(self, key, value):
return self.save(node=value)
return self.store_node_metadata(node=value)
def __delitem__(self, key):
self.remove(checksum_address=key)
@ -67,29 +80,29 @@ class NodeStorage(ABC):
def __iter__(self):
return self.all(federated_only=self.federated_only)
def _read_common_name(self, certificate: Certificate):
x509 = OpenSSL.crypto.X509.from_cryptography(certificate)
subject_components = x509.get_subject().get_components()
common_name_as_bytes = subject_components[0][1]
common_name_from_cert = common_name_as_bytes.decode()
return common_name_from_cert
@abstractmethod
def all(self, federated_only: bool) -> set:
"""Return s set of all stored nodes"""
def store_node_certificate(self,
host: str,
checksum_address: str,
certificate: Certificate,
force: bool = False
) -> str:
raise NotImplementedError
@abstractmethod
def get(self, checksum_address: str, federated_only: bool):
"""Retrieve a single stored node"""
def store_node_metadata(self, node):
"""Save a single node's metadata and tls certificate"""
raise NotImplementedError
@abstractmethod
def save(self, node):
"""Save a single node"""
raise NotImplementedError
@abstractmethod
def remove(self, checksum_address: str) -> bool:
"""Remove a single stored node"""
raise NotImplementedError
@abstractmethod
def clear(self) -> bool:
"""Remove all stored nodes"""
def generate_certificate_filepath(self, checksum_address: str) -> str:
raise NotImplementedError
@abstractmethod
@ -107,72 +120,278 @@ class NodeStorage(ABC):
"""One-time initialization steps to establish a node storage backend"""
raise NotImplementedError
@abstractmethod
def all(self, federated_only: bool, certificates_only: bool = False) -> set:
"""Return s set of all stored nodes"""
raise NotImplementedError
class InMemoryNodeStorage(NodeStorage):
@abstractmethod
def get(self, checksum_address: str, federated_only: bool):
"""Retrieve a single stored node"""
raise NotImplementedError
_name = 'memory'
@abstractmethod
def remove(self, checksum_address: str) -> bool:
"""Remove a single stored node"""
raise NotImplementedError
@abstractmethod
def clear(self) -> bool:
"""Remove all stored nodes"""
raise NotImplementedError
class ForgetfulNodeStorage(NodeStorage):
_name = ':memory:'
__base_prefix = 'nucypher-temp-cert-'
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.__known_nodes = dict()
self.__metadata = dict()
self.__certificates = dict()
def all(self, federated_only: bool) -> set:
return set(self.__known_nodes.values())
self.__rollover_certificates = list()
def get(self, checksum_address: str, federated_only: bool):
try:
return self.__known_nodes[checksum_address]
except KeyError:
raise self.UnknownNode
def all(self, federated_only: bool, certificates_only: bool = False) -> set:
return set(self.__metadata.values() if not certificates_only else self.__certificates.values())
def save(self, node):
self.__known_nodes[node.checksum_public_address] = node
return True
@validate_checksum_address
def get(self,
federated_only: bool,
host: str = None,
checksum_address: str = None,
certificate_only: bool = False):
def remove(self, checksum_address: str) -> bool:
del self.__known_nodes[checksum_address]
return True
if not bool(checksum_address) ^ bool(host):
message = "Either pass checksum_address or host; Not both. Got ({} {})".format(checksum_address, host)
raise ValueError(message)
def clear(self):
self.__known_nodes = dict()
if certificate_only is True:
try:
return self.__certificates[checksum_address or host]
except KeyError:
raise self.UnknownNode
else:
try:
return self.__metadata[checksum_address or host]
except KeyError:
raise self.UnknownNode
def forget(self, everything: bool = True) -> bool:
for temp_certificate in self.__rollover_certificates:
os.remove(temp_certificate)
if everything is True:
pattern = '/tmp/{}*'.format(self.__base_prefix)
for temp_certificate in glob.glob(pattern):
os.remove(temp_certificate)
return len(glob.glob(pattern)) == 0
return len(self.__rollover_certificates) == 0
def store_host_certificate(self, host: str, certificate: Certificate):
self.__certificates[host] = certificate
return self.generate_certificate_filepath(host=host)
@validate_checksum_address
def store_node_certificate(self,
certificate: Certificate,
checksum_address: str,
host: str = None,
force: bool = False
) -> str:
self.__certificates[checksum_address] = certificate
return self.generate_certificate_filepath(checksum_address=checksum_address)
def store_node_metadata(self, node):
self.__metadata[node.checksum_public_address] = node
return self.__metadata[node.checksum_public_address]
@validate_checksum_address
def generate_certificate_filepath(self,
checksum_address: str = None,
host: str = None) -> str:
if not bool(checksum_address) ^ bool(host):
message = "Either pass checksum_address or host; Not both. Got ({} {})".format(checksum_address, host)
raise ValueError(message)
prefix = '{}{}-'.format(self.__base_prefix, checksum_address or host)
temp_file = tempfile.NamedTemporaryFile(prefix=prefix, suffix=self.TLS_CERTIFICATE_EXTENSION, delete=False)
certificate = self.__certificates[checksum_address or host]
certificate_bytes = certificate.public_bytes(self.TLS_CERTIFICATE_ENCODING)
temp_file.write(certificate_bytes)
self.__rollover_certificates.append(temp_file.name)
return temp_file.name
@validate_checksum_address
def remove(self,
checksum_address: str,
metadata: bool = True,
certificate: bool = True
) -> Tuple[bool, str]:
if metadata is True:
del self.__metadata[checksum_address]
if certificate is True:
del self.__certificates[checksum_address]
return True, checksum_address
def clear(self, metadata: bool = True, certificates: bool = True) -> None:
"""Forget all stored nodes and certificates"""
if metadata is True:
self.__metadata = dict()
if certificates is True:
self.__certificates = dict()
def payload(self) -> dict:
payload = {self._TYPE_LABEL: self._name}
return payload
@classmethod
def from_payload(cls, payload: dict, *args, **kwargs) -> 'InMemoryNodeStorage':
def from_payload(cls, payload: dict, *args, **kwargs) -> 'ForgetfulNodeStorage':
"""Alternate constructor to create a storage instance from JSON-like configuration"""
if payload[cls._TYPE_LABEL] != cls._name:
raise cls.NodeStorageError
return cls(*args, **kwargs)
def initialize(self) -> None:
self.__known_nodes = dict()
def initialize(self) -> bool:
"""Returns True if initialization was successful"""
self.__metadata = dict()
self.__certificates = dict()
return not bool(self.__metadata or self.__certificates)
class LocalFileBasedNodeStorage(NodeStorage):
_name = 'local'
__FILENAME_TEMPLATE = '{}.node'
__DEFAULT_DIR = os.path.join(DEFAULT_CONFIG_ROOT, 'known_nodes', 'metadata')
__METADATA_FILENAME_TEMPLATE = '{}.node'
class NoNodeMetadataFileFound(FileNotFoundError, NodeStorage.UnknownNode):
pass
def __init__(self,
known_metadata_dir: str = __DEFAULT_DIR,
config_root: str = None,
storage_root: str = None,
metadata_dir: str = None,
certificates_dir: str = None,
*args, **kwargs
) -> None:
super().__init__(*args, **kwargs)
self.log = Logger(self.__class__.__name__)
self.known_metadata_dir = known_metadata_dir
def __generate_filepath(self, checksum_address: str) -> str:
metadata_path = os.path.join(self.known_metadata_dir, self.__FILENAME_TEMPLATE.format(checksum_address))
self.root_dir = storage_root
self.metadata_dir = metadata_dir
self.certificates_dir = certificates_dir
self._cache_storage_filepaths(config_root=config_root)
@staticmethod
def _generate_storage_filepaths(config_root: str = None,
storage_root: str = None,
metadata_dir: str = None,
certificates_dir: str = None):
storage_root = storage_root or os.path.join(config_root or DEFAULT_CONFIG_ROOT, 'known_nodes')
metadata_dir = metadata_dir or os.path.join(storage_root, 'metadata')
certificates_dir = certificates_dir or os.path.join(storage_root, 'certificates')
payload = {'storage_root': storage_root,
'metadata_dir': metadata_dir,
'certificates_dir': certificates_dir}
return payload
def _cache_storage_filepaths(self, config_root: str = None):
filepaths = self._generate_storage_filepaths(config_root=config_root,
storage_root=self.root_dir,
metadata_dir=self.metadata_dir,
certificates_dir=self.certificates_dir)
self.root_dir = filepaths['storage_root']
self.metadata_dir = filepaths['metadata_dir']
self.certificates_dir = filepaths['certificates_dir']
#
# Certificates
#
@validate_checksum_address
def __get_certificate_filename(self, checksum_address: str):
return '{}.{}'.format(checksum_address, Encoding.PEM.name.lower())
def __get_certificate_filepath(self, certificate_filename: str) -> str:
return os.path.join(self.certificates_dir, certificate_filename)
@validate_checksum_address
def generate_certificate_filepath(self, checksum_address: str) -> str:
certificate_filename = self.__get_certificate_filename(checksum_address)
certificate_filepath = self.__get_certificate_filepath(certificate_filename=certificate_filename)
return certificate_filepath
@validate_checksum_address
def __write_tls_certificate(self,
checksum_address: str,
certificate: Certificate,
host: str = None,
force: bool = False) -> str:
# Read
x509 = OpenSSL.crypto.X509.from_cryptography(certificate)
subject_components = x509.get_subject().get_components()
common_name_as_bytes = subject_components[0][1]
common_name_on_certificate = common_name_as_bytes.decode()
if not host:
host = common_name_on_certificate
# Validate
# TODO: It's better for us to have checked this a while ago so that this situation is impossible. #443
if host and (host != common_name_on_certificate):
raise ValueError('You passed a hostname ("{}") that does not match the certificat\'s common name.'.format(host))
certificate_filepath = self.generate_certificate_filepath(checksum_address=checksum_address)
certificate_already_exists = os.path.isfile(certificate_filepath)
if force is False and certificate_already_exists:
raise FileExistsError('A TLS certificate already exists at {}.'.format(certificate_filepath))
# Write
with open(certificate_filepath, 'wb') as certificate_file:
public_pem_bytes = certificate.public_bytes(self.TLS_CERTIFICATE_ENCODING)
certificate_file.write(public_pem_bytes)
self.certificate_filepath = certificate_filepath
self.log.info("Saved TLS certificate for {}: {}".format(self, certificate_filepath))
return certificate_filepath
@validate_checksum_address
def __read_tls_public_certificate(self, filepath: str = None, checksum_address: str=None) -> Certificate:
"""Deserialize an X509 certificate from a filepath"""
if not bool(filepath) ^ bool(checksum_address):
raise ValueError("Either pass filepath or checksum_address; Not both.")
if not filepath and checksum_address is not None:
filepath = self.generate_certificate_filepath(checksum_address)
try:
with open(filepath, 'rb') as certificate_file:
cert = x509.load_pem_x509_certificate(certificate_file.read(), backend=default_backend())
return cert
except FileNotFoundError:
raise FileNotFoundError("No SSL certificate found at {}".format(filepath))
#
# Metadata
#
@validate_checksum_address
def __generate_metadata_filepath(self, checksum_address: str) -> str:
metadata_path = os.path.join(self.metadata_dir, self.__METADATA_FILENAME_TEMPLATE.format(checksum_address))
return metadata_path
def __read(self, filepath: str, federated_only: bool):
def __read_metadata(self, filepath: str, federated_only: bool):
from nucypher.characters.lawful import Ursula
try:
with open(filepath, "rb") as seed_file:
@ -183,46 +402,107 @@ class LocalFileBasedNodeStorage(NodeStorage):
raise self.UnknownNode
return node
def __write(self, filepath: str, node):
def __write_metadata(self, filepath: str, node):
with open(filepath, "wb") as f:
f.write(self.serializer(self.character_class.__bytes__(node)))
self.log.info("Wrote new node metadata to filesystem {}".format(filepath))
return filepath
def all(self, federated_only: bool) -> set:
filenames = os.listdir(self.known_metadata_dir)
self.log.info("Found {} known node metadata files at {}".format(len(filenames), self.known_metadata_dir))
known_nodes = set()
for filename in filenames:
metadata_path = os.path.join(self.known_metadata_dir, filename)
node = self.__read(filepath=metadata_path, federated_only=federated_only) # TODO: 466
known_nodes.add(node)
return known_nodes
#
# API
#
def all(self, federated_only: bool, certificates_only: bool = False) -> Set[Union[Any, Certificate]]:
filenames = os.listdir(self.certificates_dir if certificates_only else self.metadata_dir)
self.log.info("Found {} known node metadata files at {}".format(len(filenames), self.metadata_dir))
def get(self, checksum_address: str, federated_only: bool):
metadata_path = self.__generate_filepath(checksum_address=checksum_address)
node = self.__read(filepath=metadata_path, federated_only=federated_only) # TODO: 466
known_certificates = set()
if certificates_only:
for filename in filenames:
certificate = self.__read_tls_public_certificate(os.path.join(self.certificates_dir, filename))
known_certificates.add(certificate)
return known_certificates
else:
known_nodes = set()
for filename in filenames:
metadata_path = os.path.join(self.metadata_dir, filename)
node = self.__read_metadata(filepath=metadata_path, federated_only=federated_only) # TODO: 466
known_nodes.add(node)
return known_nodes
@validate_checksum_address
def get(self, checksum_address: str, federated_only: bool, certificate_only: bool = False):
if certificate_only is True:
certificate = self.__read_tls_public_certificate(checksum_address=checksum_address)
return certificate
metadata_path = self.__generate_metadata_filepath(checksum_address=checksum_address)
node = self.__read_metadata(filepath=metadata_path, federated_only=federated_only) # TODO: 466
return node
def save(self, node):
try:
filepath = self.__generate_filepath(checksum_address=node.checksum_public_address)
except AttributeError:
raise AttributeError("{} does not have a rest_interface attached".format(self)) # TODO.. eh?
self.__write(filepath=filepath, node=node)
@validate_checksum_address
def store_node_certificate(self,
checksum_address: str,
certificate: Certificate,
host: str = None,
force: bool = True
) -> str:
def remove(self, checksum_address: str):
filepath = self.__generate_filepath(checksum_address=checksum_address)
self.log.debug("Delted {} from the filesystem".format(checksum_address))
return os.remove(filepath)
certificate_filepath = self.__write_tls_certificate(certificate=certificate,
checksum_address=checksum_address,
host=host,
force=force)
def clear(self):
self.__known_nodes = dict()
return certificate_filepath
def store_node_metadata(self, node) -> str:
filepath = self.__generate_metadata_filepath(checksum_address=node.checksum_public_address)
self.__write_metadata(filepath=filepath, node=node)
return filepath
def save_node(self, node, force) -> Tuple[str, str]:
certificate_filepath = self.store_node_certificate(checksum_address=node.checksum_public_address,
certificate=node.certificate,
force=force)
metadata_filepath = self.store_node_metadata(node=node)
return metadata_filepath, certificate_filepath
@validate_checksum_address
def remove(self, checksum_address: str, metadata: bool = True, certificate: bool = True) -> None:
if metadata is True:
metadata_filepath = self.__generate_metadata_filepath(checksum_address=checksum_address)
os.remove(metadata_filepath)
self.log.debug("Deleted {} from the filesystem".format(checksum_address))
if certificate is True:
certificate_filepath = self.generate_certificate_filepath(checksum_address=checksum_address)
os.remove(certificate_filepath)
self.log.debug("Deleted {} from the filesystem".format(checksum_address))
return
def clear(self, metadata: bool = True, certificates: bool = True) -> None:
"""Forget all stored nodes and certificates"""
def __destroy_dir_contents(path):
for file in os.listdir(path):
file_path = os.path.join(path, file)
if os.path.isfile(file_path):
os.unlink(file_path)
if metadata is True:
__destroy_dir_contents(self.metadata_dir)
if certificates is True:
__destroy_dir_contents(self.certificates_dir)
return
def payload(self) -> dict:
payload = {
'storage_type': self._name,
'known_metadata_dir': self.known_metadata_dir
'storage_root': self.root_dir,
'metadata_dir': self.metadata_dir,
'certificates_dir': self.certificates_dir
}
return payload
@ -231,35 +511,55 @@ class LocalFileBasedNodeStorage(NodeStorage):
storage_type = payload[cls._TYPE_LABEL]
if not storage_type == cls._name:
raise cls.NodeStorageError("Wrong storage type. got {}".format(storage_type))
return cls(known_metadata_dir=payload['known_metadata_dir'], *args, **kwargs)
del payload['storage_type']
def initialize(self):
return cls(*args, **payload, **kwargs)
def initialize(self) -> bool:
try:
os.mkdir(self.known_metadata_dir, mode=0o755) # known_metadata
os.mkdir(self.root_dir, mode=0o755)
os.mkdir(self.metadata_dir, mode=0o755)
os.mkdir(self.certificates_dir, mode=0o755)
except FileExistsError:
message = "There are pre-existing metadata files at {}".format(self.known_metadata_dir)
message = "There are pre-existing files at {}".format(self.root_dir)
raise self.NodeStorageError(message)
except FileNotFoundError:
raise self.NodeStorageError("There is no existing configuration at {}".format(self.known_metadata_dir))
raise self.NodeStorageError("There is no existing configuration at {}".format(self.root_dir))
return bool(all(map(os.path.isdir, (self.root_dir, self.metadata_dir, self.certificates_dir))))
class TemporaryFileBasedNodeStorage(LocalFileBasedNodeStorage):
_name = 'tmp'
def __init__(self, *args, **kwargs):
self.__temp_dir = constants.NO_STORAGE_AVAILIBLE
super().__init__(known_metadata_dir=self.__temp_dir, *args, **kwargs)
self.__temp_metadata_dir = None
self.__temp_certificates_dir = None
super().__init__(metadata_dir=self.__temp_metadata_dir,
certificates_dir=self.__temp_certificates_dir,
*args, **kwargs)
def __del__(self):
if not self.__temp_dir is constants.NO_STORAGE_AVAILIBLE:
shutil.rmtree(self.__temp_dir, ignore_errors=True)
if self.__temp_metadata_dir is not None:
shutil.rmtree(self.__temp_metadata_dir, ignore_errors=True)
shutil.rmtree(self.__temp_certificates_dir, ignore_errors=True)
def initialize(self):
self.__temp_dir = tempfile.mkdtemp(prefix="nucypher-tmp-nodes-")
self.known_metadata_dir = self.__temp_dir
def initialize(self) -> bool:
# Metadata
self.__temp_metadata_dir = tempfile.mkdtemp(prefix="nucypher-tmp-nodes-")
self.metadata_dir = self.__temp_metadata_dir
# Certificates
self.__temp_certificates_dir = tempfile.mkdtemp(prefix="nucypher-tmp-certs-")
self.certificates_dir = self.__temp_certificates_dir
return bool(os.path.isdir(self.metadata_dir) and os.path.isdir(self.certificates_dir))
class S3NodeStorage(NodeStorage):
_name = 's3'
S3_ACL = 'private' # Canned S3 Permissions
def __init__(self,
@ -271,7 +571,7 @@ class S3NodeStorage(NodeStorage):
self.__bucket_name = bucket_name
self.__s3client = boto3.client('s3')
self.__s3resource = s3_resource or boto3.resource('s3')
self.__bucket = constants.NO_STORAGE_AVAILIBLE
self.__bucket = NO_STORAGE_AVAILIBLE
@property
def bucket(self):
@ -290,12 +590,13 @@ class S3NodeStorage(NodeStorage):
node = self.character_class.from_bytes(node_bytes)
return node
@validate_checksum_address
def generate_presigned_url(self, checksum_address: str) -> str:
payload = {'Bucket': self.__bucket_name, 'Key': checksum_address}
url = self.__s3client.generate_presigned_url('get_object', payload, ExpiresIn=900)
return url
def all(self, federated_only: bool) -> set:
def all(self, federated_only: bool, certificates_only: bool = False) -> set:
node_objs = self.__bucket.objects.all()
nodes = set()
for node_obj in node_objs:
@ -303,17 +604,19 @@ class S3NodeStorage(NodeStorage):
nodes.add(node)
return nodes
@validate_checksum_address
def get(self, checksum_address: str, federated_only: bool):
node_obj = self.__bucket.Object(checksum_address)
node = self.__read(node_obj=node_obj)
return node
def save(self, node):
def store_node_metadata(self, node):
self.__s3client.put_object(Bucket=self.__bucket_name,
ACL=self.S3_ACL,
Key=node.checksum_public_address,
Body=self.serializer(bytes(node)))
@validate_checksum_address
def remove(self, checksum_address: str) -> bool:
node_obj = self.__bucket.Object(checksum_address)
response = node_obj.delete()

243
nucypher/network/nodes.py
View File

@ -14,20 +14,19 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import binascii
import os
import random
from collections import defaultdict, OrderedDict
from collections import deque
from collections import namedtuple
from contextlib import suppress
from logging import Logger
from tempfile import TemporaryDirectory
from typing import Set, Tuple
import OpenSSL
import maya
import requests
import socket
import time
from bytestring_splitter import BytestringSplitter, VariableLengthBytestring, BytestringSplittingError
from constant_sorrow import constants
@ -39,19 +38,22 @@ from twisted.internet import reactor, defer
from twisted.internet import task
from twisted.internet.threads import deferToThread
from twisted.logger import Logger
from typing import Set, Tuple
from bytestring_splitter import BytestringSplitter
from constant_sorrow import constants
from constant_sorrow.constants import constant_or_bytes, GLOBAL_DOMAIN
from nucypher.config.constants import SeednodeMetadata
from nucypher.config.keyring import _write_tls_certificate
from nucypher.config.storages import InMemoryNodeStorage
from nucypher.config.storages import ForgetfulNodeStorage
from nucypher.crypto.api import keccak_digest
from nucypher.crypto.powers import BlockchainPower, SigningPower, EncryptingPower, NoSigningPower
from nucypher.crypto.signing import signature_splitter
from nucypher.network import LEARNING_LOOP_VERSION
from nucypher.network.middleware import RestMiddleware
from nucypher.network.nicknames import nickname_from_seed
from nucypher.network.protocols import SuspiciousActivity
from nucypher.network.protocols import SuspiciousActivity, parse_node_uri
from nucypher.network.server import TLSHostingPower
from nucypher.utilities.decorators import validate_checksum_address
def icon_from_checksum(checksum,
@ -144,10 +146,16 @@ class FleetStateTracker:
def nickname_metadata(self):
return self._nickname_metadata
@property
def icon(self) -> str:
if self.nickname_metadata is constants.NO_KNOWN_NODES:
return str(constants.NO_KNOWN_NODES)
return self.nickname_metadata[0][1]
def addresses(self):
return self._nodes.keys()
def icon(self):
def icon_html(self):
return icon_from_checksum(checksum=self.checksum,
number_of_nodes=len(self),
nickname_metadata=self.nickname_metadata)
@ -173,11 +181,18 @@ class FleetStateTracker:
# For now we store the sorted node list. Someday we probably spin this out into
# its own class, FleetState, and use it as the basis for partial updates.
self.states[checksum] = self.state_template(nickname=self.nickname,
nodes=sorted_nodes,
icon=self.icon_html(),
icon=self.icon(),
nodes=sorted_nodes,
updated=self.updated,
)
def start_tracking_state(self, additional_nodes_to_track=[]):
self.additional_nodes_to_track.extend(additional_nodes_to_track)
self._tracking = True
self.update_fleet_state()
def sorted(self):
nodes_to_consider = list(self._nodes.values()) + self.additional_nodes_to_track
return sorted(nodes_to_consider, key=lambda n: n.checksum_public_address)
@ -202,7 +217,7 @@ class Learner:
_ROUNDS_WITHOUT_NODES_AFTER_WHICH_TO_SLOW_DOWN = 10
# For Keeps
__DEFAULT_NODE_STORAGE = InMemoryNodeStorage
__DEFAULT_NODE_STORAGE = ForgetfulNodeStorage
__DEFAULT_MIDDLEWARE_CLASS = RestMiddleware
LEARNER_VERSION = LEARNING_LOOP_VERSION
@ -226,7 +241,6 @@ class Learner:
learn_on_same_thread: bool = False,
known_nodes: tuple = None,
seed_nodes: Tuple[tuple] = None,
known_certificates_dir: str = None,
node_storage=None,
save_metadata: bool = False,
abort_on_learning_error: bool = False
@ -244,7 +258,6 @@ class Learner:
self._learning_listeners = defaultdict(list)
self._node_ids_to_learn_about_immediately = set()
self.known_certificates_dir = known_certificates_dir or TemporaryDirectory("nucypher-tmp-certs-").name
self.__known_nodes = FleetStateTracker()
self.done_seeding = False
@ -263,8 +276,7 @@ class Learner:
self.unresponsive_startup_nodes = list() # TODO: Attempt to use these again later
for node in known_nodes:
try:
self.remember_node(
node) # TODO: Need to test this better - do we ever init an Ursula-Learner with Node Storage?
self.remember_node(node) # TODO: Need to test this better - do we ever init an Ursula-Learner with Node Storage?
except self.UnresponsiveTeacher:
self.unresponsive_startup_nodes.append(node)
@ -298,14 +310,12 @@ class Learner:
def __attempt_seednode_learning(seednode_metadata, current_attempt=1):
from nucypher.characters.lawful import Ursula
self.log.debug(
"Seeding from: {}|{}:{}".format(seednode_metadata.checksum_address,
"Seeding from: {}|{}:{}".format(seednode_metadata.checksum_public_address,
seednode_metadata.rest_host,
seednode_metadata.rest_port))
seed_node = Ursula.from_seednode_metadata(seednode_metadata=seednode_metadata,
network_middleware=self.network_middleware,
certificates_directory=self.known_certificates_dir,
timeout=timeout,
federated_only=self.federated_only) # TODO: 466
if seed_node is False:
self.unresponsive_seed_nodes.add(seednode_metadata)
@ -345,8 +355,11 @@ class Learner:
# This node is already known. We can safely return.
return False
node.save_certificate_to_disk(directory=self.known_certificates_dir, force=True) # TODO: Verify before force?
certificate_filepath = node.get_certificate_filepath(certificates_dir=self.known_certificates_dir)
# Store node's certificate - It has been seen.
certificate_filepath = self.node_storage.store_node_certificate(checksum_address=node.checksum_public_address,
certificate=node.certificate,
host=node.rest_information()[0].host)
try:
node.verify_node(force=force_verification_check,
network_middleware=self.network_middleware,
@ -354,6 +367,7 @@ class Learner:
certificate_filepath=certificate_filepath)
except SSLError:
return False # TODO: Bucket this node as having bad TLS info - maybe it's an update that hasn't fully propagated?
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
self.log.info("No Response while trying to verify node {}|{}".format(node.rest_interface, node))
return False # TODO: Bucket this node as "ghost" or something: somebody else knows about it, but we can't get to it.
@ -366,7 +380,7 @@ class Learner:
raise RuntimeError
if self.save_metadata:
self.write_node_metadata(node=node)
self.node_storage.store_node_metadata(node=node)
self.log.info("Remembering {}, popping {} listeners.".format(node.checksum_public_address, len(listeners)))
for listener in listeners:
@ -593,7 +607,7 @@ class Learner:
# Scenario 3: We don't know about this node, and neither does our friend.
def write_node_metadata(self, node, serializer=bytes) -> str:
return self.node_storage.save(node=node)
return self.node_storage.store_node_metadata(node=node)
def learn_from_teacher_node(self, eager=True):
"""
@ -617,6 +631,8 @@ class Learner:
unresponsive_nodes = set()
try:
# TODO: Streamline path generation
certificate_filepath = self.node_storage.generate_certificate_filepath(checksum_address=current_teacher.checksum_public_address)
response = self.network_middleware.get_nodes_via_rest(url=rest_url,
certificate_filepath = current_teacher.get_certificate_filepath(
certificates_dir=self.known_certificates_dir)
response = self.network_middleware.get_nodes_via_rest(url=teacher_uri,
@ -669,8 +685,7 @@ class Learner:
continue # This node is not serving any of our domains.
try:
if eager:
certificate_filepath = current_teacher.get_certificate_filepath(
certificates_dir=self.known_certificates_dir)
certificate_filepath = self.node_storage.generate_certificate_filepath(checksum_address=current_teacher.checksum_public_address)
node.verify_node(self.network_middleware,
accept_federated_only=self.federated_only, # TODO: 466
certificate_filepath=certificate_filepath)
@ -712,7 +727,8 @@ class Teacher:
verified_interface = False
_verified_node = False
_interface_info_splitter = (int, 4, {'byteorder': 'big'})
log = Logger("network/nodes")
log = Logger("teacher")
__DEFAULT_MIN_SEED_STAKE = 0
def __init__(self,
domains: Set,
@ -753,17 +769,21 @@ class Teacher:
Raised when deserializing a Character from a future version.
"""
def seed_node_metadata(self):
return SeednodeMetadata(self.checksum_public_address,
self.rest_server.rest_interface.host,
self.rest_server.rest_interface.port)
@classmethod
def from_tls_hosting_power(cls, tls_hosting_power: TLSHostingPower, *args, **kwargs) -> 'Teacher':
certificate_filepath = tls_hosting_power.keypair.certificate_filepath
certificate = tls_hosting_power.keypair.certificate
return cls(certificate=certificate, certificate_filepath=certificate_filepath, *args, **kwargs)
#
# Known Nodes
#
def seed_node_metadata(self):
return SeednodeMetadata(self.checksum_public_address, # type: str
self.rest_server.rest_interface.host, # type: str
self.rest_server.rest_interface.port) # type: int
def sorted_nodes(self):
nodes_to_consider = list(self.known_nodes.values()) + [self]
return sorted(nodes_to_consider, key=lambda n: n.checksum_public_address)
@ -787,6 +807,19 @@ class Teacher:
self.fleet_state_updated = updated
self.fleet_state_icon = icon_from_checksum(self.fleet_state_checksum,
nickname_metadata=self.fleet_state_nickname_metadata)
#
# Stamp
#
def _stamp_has_valid_wallet_signature(self):
signature_bytes = self._evidence_of_decentralized_identity
if signature_bytes is constants.NOT_SIGNED:
return False
else:
signature = EthSignature(signature_bytes)
proper_pubkey = signature.recover_public_key_from_msg(bytes(self.stamp))
proper_address = proper_pubkey.to_checksum_address()
return proper_address == self.checksum_public_address
def stamp_is_valid(self):
"""
@ -804,19 +837,6 @@ class Teacher:
else:
raise self.InvalidNode
def interface_is_valid(self):
"""
Checks that the interface info is valid for this node's canonical address.
"""
interface_info_message = self._signable_interface_info_message() # Contains canonical address.
message = self.timestamp_bytes() + interface_info_message
interface_is_valid = self._interface_signature.verify(message, self.public_keys(SigningPower))
self.verified_interface = interface_is_valid
if interface_is_valid:
return True
else:
raise self.InvalidNode
def verify_id(self, ursula_id, digest_factory=bytes):
self.verify()
if not ursula_id == digest_factory(self.canonical_public_address):
@ -879,12 +899,29 @@ class Teacher:
else:
self._verified_node = True
def substantiate_stamp(self, passphrase: str):
def substantiate_stamp(self, password: str):
blockchain_power = self._crypto_power.power_ups(BlockchainPower)
blockchain_power.unlock_account(password=passphrase) # TODO: 349
blockchain_power.unlock_account(password=password) # TODO: 349
signature = blockchain_power.sign_message(bytes(self.stamp))
self._evidence_of_decentralized_identity = signature
#
# Interface
#
def interface_is_valid(self):
"""
Checks that the interface info is valid for this node's canonical address.
"""
interface_info_message = self._signable_interface_info_message() # Contains canonical address.
message = self.timestamp_bytes() + interface_info_message
interface_is_valid = self._interface_signature.verify(message, self.public_keys(SigningPower))
self.verified_interface = interface_is_valid
if interface_is_valid:
return True
else:
raise self.InvalidNode
def _signable_interface_info_message(self):
message = self.canonical_public_address + self.rest_information()[0]
return message
@ -915,123 +952,15 @@ class Teacher:
def timestamp_bytes(self):
return self.timestamp.epoch.to_bytes(4, 'big')
@property
def common_name(self):
x509 = OpenSSL.crypto.X509.from_cryptography(self.certificate)
subject_components = x509.get_subject().get_components()
common_name_as_bytes = subject_components[0][1]
common_name_from_cert = common_name_as_bytes.decode()
return common_name_from_cert
#
# Nicknames
#
@property
def certificate_filename(self):
return '{}.{}'.format(self.checksum_public_address, Encoding.PEM.name.lower()) # TODO: use cert's encoding..?
def get_certificate_filepath(self, certificates_dir: str) -> str:
return os.path.join(certificates_dir, self.certificate_filename)
def save_certificate_to_disk(self, directory, force=False):
x509 = OpenSSL.crypto.X509.from_cryptography(self.certificate)
subject_components = x509.get_subject().get_components()
common_name_as_bytes = subject_components[0][1]
common_name_from_cert = common_name_as_bytes.decode()
if not self.rest_information()[0].host == common_name_from_cert:
# TODO: It's better for us to have checked this a while ago so that this situation is impossible. #443
raise ValueError("You passed a common_name that is not the same one as the cert. "
"Common name is optional; the cert will be saved according to "
"the name on the cert itself.")
certificate_filepath = self.get_certificate_filepath(certificates_dir=directory)
_write_tls_certificate(self.certificate, full_filepath=certificate_filepath, force=force)
self.certificate_filepath = certificate_filepath
self.log.info("Saved TLS certificate for {}: {}".format(self, certificate_filepath))
@classmethod
def from_seednode_metadata(cls,
seednode_metadata,
*args,
**kwargs):
"""
Essentially another deserialization method, but this one doesn't reconstruct a complete
node from bytes; instead it's just enough to connect to and verify a node.
"""
return cls.from_seed_and_stake_info(checksum_address=seednode_metadata.checksum_address,
host=seednode_metadata.rest_host,
port=seednode_metadata.rest_port,
*args, **kwargs)
@classmethod
def from_seed_and_stake_info(cls, host,
certificates_directory,
federated_only,
port=9151,
checksum_address=None,
minimum_stake=0,
network_middleware=None,
*args,
**kwargs
):
if network_middleware is None:
network_middleware = RestMiddleware()
certificate = network_middleware.get_certificate(host=host, port=port)
real_host = certificate.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
# Write certificate; this is really only for temporary purposes. Ideally, we'd use
# it in-memory here but there's no obvious way to do that.
filename = '{}.{}'.format(checksum_address, Encoding.PEM.name.lower())
certificate_filepath = os.path.join(certificates_directory, filename)
_write_tls_certificate(certificate=certificate, full_filepath=certificate_filepath, force=True)
cls.log.info("Saved seednode {} TLS certificate".format(checksum_address))
potential_seed_node = cls.from_rest_url(
host=real_host,
port=port,
network_middleware=network_middleware,
certificate_filepath=certificate_filepath,
federated_only=True,
*args,
**kwargs) # TODO: 466
if checksum_address:
if not checksum_address == potential_seed_node.checksum_public_address:
raise potential_seed_node.SuspiciousActivity(
"This seed node has a different wallet address: {} (was hoping for {}). Are you sure this is a seed node?".format(
potential_seed_node.checksum_public_address,
checksum_address))
else:
if minimum_stake > 0:
# TODO: check the blockchain to verify that address has more then minimum_stake. #511
raise NotImplementedError("Stake checking is not implemented yet.")
try:
potential_seed_node.verify_node(
network_middleware=network_middleware,
accept_federated_only=federated_only,
certificate_filepath=certificate_filepath)
except potential_seed_node.InvalidNode:
raise # TODO: What if our seed node fails verification?
return potential_seed_node
@classmethod
def from_rest_url(cls,
network_middleware: RestMiddleware,
host: str,
port: int,
certificate_filepath,
federated_only: bool = False,
*args,
**kwargs):
response = network_middleware.node_information(host, port, certificate_filepath=certificate_filepath)
if not response.status_code == 200:
raise RuntimeError("Got a bad response: {}".format(response))
stranger_ursula_from_public_keys = cls.from_bytes(response.content, federated_only=federated_only)
return stranger_ursula_from_public_keys
def nickname_icon(self):
return '{} {}'.format(self.nickname_metadata[0][1], self.nickname_metadata[1][1])
def nickname_icon_html(self):
icon_template = """
<div class="nucypher-nickname-icon" style="border-top-color:{first_color}; border-left-color:{first_color}; border-bottom-color:{second_color}; border-right-color:{second_color};">
<span class="small">{node_class} v{version}</span>

26
nucypher/network/protocols.py
View File

@ -14,6 +14,12 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from urllib.parse import urlparse
from eth_utils import is_checksum_address
from bytestring_splitter import VariableLengthBytestring
@ -21,6 +27,26 @@ class SuspiciousActivity(RuntimeError):
"""raised when an action appears to amount to malicious conduct."""
def parse_node_uri(uri: str):
from nucypher.config.characters import UrsulaConfiguration
if '@' in uri:
checksum_address, uri = uri.split("@")
if not is_checksum_address(checksum_address):
raise ValueError("{} is not a valid checksum address.".format(checksum_address))
else:
checksum_address = None # federated
# HTTPS Explicit Required
parsed_uri = urlparse(uri)
if not parsed_uri.scheme == "https":
raise ValueError("Invalid teacher URI. Is the hostname prefixed with 'https://' ?")
hostname = parsed_uri.hostname
port = parsed_uri.port or UrsulaConfiguration.DEFAULT_REST_PORT
return hostname, port, checksum_address
class InterfaceInfo:
expected_bytes_length = lambda: VariableLengthBytestring

100
nucypher/network/server.py
View File

@ -18,6 +18,8 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
import binascii
import os
from typing import Callable
from twisted.logger import Logger
from apistar import Route, App
@ -26,6 +28,9 @@ from bytestring_splitter import VariableLengthBytestring
from constant_sorrow import constants
from constant_sorrow.constants import GLOBAL_DOMAIN
from hendrix.experience import crosstown_traffic
from nucypher.config.storages import ForgetfulNodeStorage
from nucypher.crypto.signing import SignatureStamp
from nucypher.network.middleware import RestMiddleware
from umbral import pre
from umbral.fragments import KFrag
from umbral.keys import UmbralPublicKey
@ -39,15 +44,16 @@ from nucypher.keystore.keystore import NotFound
from nucypher.keystore.threading import ThreadedSession
from nucypher.network import LEARNING_LOOP_VERSION
from nucypher.network.protocols import InterfaceInfo
from jinja2 import Template
from jinja2 import Template, TemplateError
HERE = BASE_DIR = os.path.abspath(os.path.dirname(__file__))
TEMPLATES_DIR = os.path.join(HERE, "templates")
class ProxyRESTServer:
log = Logger("characters")
SERVER_VERSION = LEARNING_LOOP_VERSION
log = Logger("network-server")
def __init__(self,
rest_host: str,
@ -71,27 +77,28 @@ class ProxyRESTServer:
class ProxyRESTRoutes:
log = Logger("characters")
log = Logger("network-server")
def __init__(self,
db_name,
db_filepath,
network_middleware,
federated_only,
treasure_map_tracker,
node_tracker,
node_bytes_caster,
work_order_tracker,
node_recorder,
stamp,
verifier,
suspicious_activity_tracker,
certificate_dir,
db_filepath: str,
network_middleware: RestMiddleware,
federated_only: bool,
treasure_map_tracker: dict,
node_tracker: 'FleetStateTracker',
node_bytes_caster: Callable,
work_order_tracker: list,
node_recorder: Callable,
stamp: SignatureStamp,
verifier: Callable,
suspicious_activity_tracker: dict,
serving_domains,
) -> None:
self.network_middleware = network_middleware
self.federated_only = federated_only
self.datastore = None
self.__forgetful_node_storage = ForgetfulNodeStorage(federated_only=federated_only)
self._treasure_map_tracker = treasure_map_tracker
self._work_order_tracker = work_order_tracker
@ -136,7 +143,6 @@ class ProxyRESTRoutes:
]
self.rest_app = App(routes=routes)
self.db_name = db_name
self.db_filepath = db_filepath
from nucypher.keystore import keystore
@ -144,7 +150,11 @@ class ProxyRESTRoutes:
from sqlalchemy.engine import create_engine
self.log.info("Starting datastore {}".format(self.db_filepath))
engine = create_engine('sqlite:///{}'.format(self.db_filepath))
# See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings
db_filepath = (self.db_filepath or '') # Capture None
engine = create_engine('sqlite:///{}'.format(db_filepath))
Base.metadata.create_all(engine)
self.datastore = keystore.KeyStore(engine)
self.db_engine = engine
@ -190,11 +200,10 @@ class ProxyRESTRoutes:
signature = self._stamp(payload)
return Response(bytes(signature) + payload, headers=headers, status_code=204)
nodes = self._node_class.batch_from_bytes(request.body,
federated_only=self.federated_only, # TODO: 466
)
nodes = self._node_class.batch_from_bytes(request.body, federated_only=self.federated_only) # TODO: 466
# TODO: This logic is basically repeated in learn_from_teacher_node and remember_node. Let's find a better way. 555
# TODO: This logic is basically repeated in learn_from_teacher_node and remember_node.
# Let's find a better way. #555
for node in nodes:
if GLOBAL_DOMAIN not in self.serving_domains:
if not self.serving_domains.intersection(node.serving_domains):
@ -206,23 +215,35 @@ class ProxyRESTRoutes:
@crosstown_traffic()
def learn_about_announced_nodes():
try:
certificate_filepath = node.get_certificate_filepath(certificates_dir=self._certificate_dir) # TODO: integrate with recorder?
node.save_certificate_to_disk(directory=self._certificate_dir, force=True)
temp_certificate_filepath = self.__forgetful_node_storage.store_node_certificate(checksum_address=node.checksum_public_address,
certificate=node.certificate)
node.verify_node(self.network_middleware,
accept_federated_only=self.federated_only, # TODO: 466
certificate_filepath=certificate_filepath)
certificate_filepath=temp_certificate_filepath)
# Suspicion
except node.SuspiciousActivity:
# TODO: Include data about caller?
# TODO: Account for possibility that stamp, rather than interface, was bad.
message = "Suspicious Activity: Discovered node with bad signature: {}. " \
" Announced via REST." # TODO: Include data about caller?
# TODO: Maybe also record the bytes representation separately to disk?
message = "Suspicious Activity: Discovered node with bad signature: {}. Announced via REST."
self.log.warn(message)
self._suspicious_activity_tracker['vladimirs'].append(node) # TODO: Maybe also record the bytes representation separately to disk?
self._suspicious_activity_tracker['vladimirs'].append(node)
# Async Sentinel
except Exception as e:
self.log.critical(str(e))
raise # TODO
raise
# Believable
else:
self.log.info("Learned about previously unknown node: {}".format(node))
self._node_recorder(node)
# TODO: Record new fleet state
# Cleanup
finally:
self.__forgetful_node_storage.forget(everything=True)
# TODO: What's the right status code here? 202? Different if we already knew about the node?
return self.all_known_nodes(request)
@ -391,13 +412,22 @@ class ProxyRESTRoutes:
assert False
def status(self, request: Request):
headers = {"Content-Type": "text/html", "charset":"utf-8"}
# TODO: Seems very strange to deserialize *this node* when we can just pass it in. Might be a sign that we need to rethnk this composition.
# TODO: Seems very strange to deserialize *this node* when we can just pass it in.
# Might be a sign that we need to rethnk this composition.
headers = {"Content-Type": "text/html", "charset": "utf-8"}
this_node = self._node_class.from_bytes(self._node_bytes_caster(), federated_only=self.federated_only)
content = self._status_template.render(known_nodes=self._node_tracker,
this_node=this_node,
domains=[str(d) for d in self.serving_domains],
previous_states=list(reversed(self._node_tracker.states.values()))[:5])
previous_states = list(reversed(self._node_tracker.states.values()))[:5]
try:
content = self._status_template.render(this_node=this_node,
known_nodes=self._node_tracker,
previous_states=previous_states)
except Exception as e:
self.log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(content=content, headers=headers)

35
nucypher/network/templates/basic_status.j2
View File

@ -66,6 +66,36 @@
font-size: 2em;
}
#known-nodes {
float:left;
clear:left;
}
.small-address {
text-shadow: none;
}
.state {
float:left;
}
#previous-states {
float:left;
clear:left;
}
#previous-states .state {
margin:left: 10px;
border-right: 3px solid black;
}
#previous-states .nucypher-nickname-icon {
height:75px;
width: 75px;
}
#previous-states .single-symbol {
font-size: 2em;
}
#known-nodes {
float:left;
clear:left;
@ -74,13 +104,14 @@
<div id="this-node">
<h2>{{ this_node.nickname}}</h2>
{{ this_node.nickname_icon }}
{{ this_node.nickname_icon() }}
<h4>Domains: {% for domain in domains %}{{ domain }} {% endfor %}</h4>
<h3>Fleet State</h3>
<div class="state">
<h4>{{ known_nodes.nickname }}</h4>
{{ known_nodes.icon() }}
{{ known_nodes.icon }}
<br/>
<span class="small">{{ known_nodes.updated }}</span>
</ul>
@ -110,7 +141,7 @@
</thead>
{% for node in known_nodes -%}
<tr>
<td>{{ node.nickname_icon() }}</td>
<td>{{ node.nickname_icon }}</td>
<td>
<a href="https://{{ node.rest_url()}}/status">{{ node.nickname }}</a>
<br/><span class="small">{{ node.checksum_public_address }}</span>

2
nucypher/policy/models.py
View File

@ -256,7 +256,7 @@ class Policy:
return self.publish(network_middleware)
def consider_arrangement(self, network_middleware, ursula, arrangement):
certificate_filepath = ursula.get_certificate_filepath(certificates_dir=self.alice.known_certificates_dir)
certificate_filepath = ursula.node_storage.generate_certificate_filepath(checksum_address=arrangement.ursula.checksum_public_address)
try:
ursula.verify_node(network_middleware,
accept_federated_only=arrangement.federated,

65
nucypher/utilities/decorators.py Normal file
View File

@ -0,0 +1,65 @@
import functools
from twisted.logger import Logger
from typing import Callable
import inspect
import eth_utils
def validate_checksum_address(func: Callable) -> Callable:
"""
EIP-55 Checksum address validation decorator.
Inspects the decorated function for an input parameter "checksum_address",
then uses `eth_utils` to validate the address EIP-55 checksum,
verifying the input type on failure; Raises TypeError
or InvalidChecksumAddress if validation fails, respectively.
EIP-55 Specification: https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md
ETH Utils Implementation: https://github.com/ethereum/eth-utils
"""
parameter_name = 'checksum_address'
log = Logger('EIP-55-validator')
class InvalidChecksumAddress(eth_utils.exceptions.ValidationError):
pass
@functools.wraps(func)
def wrapped(*args, **kwargs):
# Check for the presence of a checksum address in this call
params = inspect.getcallargs(func, *args, **kwargs)
try:
checksum_address = params[parameter_name]
# No checksum_address present in this call
except KeyError:
return func(*args, **kwargs) # ... don't mind me!
# Optional checksum_address present in this call
signature = inspect.signature(func)
checksum_address_is_optional = signature.parameters[parameter_name].default is None
if checksum_address_is_optional and checksum_address is None:
return func(*args, **kwargs) # ... nothing to validate
# Validate
address_is_valid = eth_utils.is_checksum_address(checksum_address)
# OK!
if address_is_valid:
return func(*args, **kwargs)
# Invalid Type
if not isinstance(checksum_address, str):
actual_type_name = checksum_address.__class__.__name__
message = '{} is an invalid type for parameter "{}".'.format(actual_type_name, parameter_name)
raise TypeError(message)
# Invalid Value
message = '"{}" is not a valid EIP-55 checksum address.'.format(checksum_address)
log.debug(message)
raise InvalidChecksumAddress(message)
return wrapped

17
nucypher/utilities/logging.py
View File

@ -19,15 +19,32 @@ import datetime
import pathlib
from sentry_sdk import capture_exception, add_breadcrumb
from sentry_sdk.integrations.logging import LoggingIntegration
from twisted.logger import FileLogObserver, jsonFileLogObserver
from twisted.logger import ILogObserver
from twisted.logger import LogLevel
from twisted.python.logfile import DailyLogFile
from zope.interface import provider
import nucypher
from nucypher.config.constants import USER_LOG_DIR
def initialize_sentry(dsn: str):
import sentry_sdk
import logging
sentry_logging = LoggingIntegration(
level=logging.INFO, # Capture info and above as breadcrumbs
event_level=logging.DEBUG # Send debug logs as events
)
sentry_sdk.init(
dsn=dsn,
integrations=[sentry_logging],
release=nucypher.__version__
)
def formatUrsulaLogEvent(event):
"""
Format log lines for file logging.

16
nucypher/utilities/sandbox/blockchain.py
View File

@ -24,8 +24,8 @@ from web3.middleware import geth_poa_middleware
from nucypher.blockchain.eth import constants
from nucypher.blockchain.eth.chains import Blockchain
from nucypher.utilities.sandbox.constants import (DEVELOPMENT_ETH_AIRDROP_AMOUNT,
DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD)
NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
INSECURE_DEVELOPMENT_PASSWORD)
def token_airdrop(token_agent, amount: int, origin: str, addresses: List[str]):
@ -62,11 +62,11 @@ class TesterBlockchain(Blockchain):
w3.middleware_stack.inject(geth_poa_middleware, layer=0)
# Generate additional ethereum accounts for testing
enough_accounts = len(self.interface.w3.eth.accounts) >= DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
enough_accounts = len(self.interface.w3.eth.accounts) >= NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
if test_accounts is not None and not enough_accounts:
accounts_to_make = DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK - len(self.interface.w3.eth.accounts)
test_accounts = test_accounts if test_accounts is not None else DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
accounts_to_make = NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK - len(self.interface.w3.eth.accounts)
test_accounts = test_accounts if test_accounts is not None else NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
self.__generate_insecure_unlocked_accounts(quantity=accounts_to_make)
@ -84,14 +84,14 @@ class TesterBlockchain(Blockchain):
Generate additional unlocked accounts transferring a balance to each account on creation.
"""
addresses = list()
insecure_passphrase = TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
insecure_password = INSECURE_DEVELOPMENT_PASSWORD
for _ in range(quantity):
umbral_priv_key = UmbralPrivateKey.gen_key()
address = self.interface.w3.personal.importRawKey(private_key=umbral_priv_key.to_bytes(),
passphrase=insecure_passphrase)
password=insecure_password)
assert self.interface.unlock_account(address, password=insecure_passphrase, duration=None), 'Failed to unlock {}'.format(address)
assert self.interface.unlock_account(address, password=insecure_password, duration=None), 'Failed to unlock {}'.format(address)
addresses.append(address)
self._test_account_cache.append(address)
self.log.info('Generated new insecure account {}'.format(address))

21
nucypher/utilities/sandbox/constants.py
View File

@ -14,16 +14,19 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
from nucypher.blockchain.eth.constants import DISPATCHER_SECRET_LENGTH, M
from nucypher.config.constants import DEFAULT_CONFIG_ROOT
TEST_KNOWN_URSULAS_CACHE = {}
TEST_URSULA_STARTING_PORT = 7468
MOCK_KNOWN_URSULAS_CACHE = {}
DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK = 10
MOCK_URSULA_STARTING_PORT = 49152
NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK = 10
DEVELOPMENT_TOKEN_AIRDROP_AMOUNT = 1000000 * int(M)
@ -33,6 +36,14 @@ MINERS_ESCROW_DEPLOYMENT_SECRET = os.urandom(DISPATCHER_SECRET_LENGTH)
POLICY_MANAGER_DEPLOYMENT_SECRET = os.urandom(DISPATCHER_SECRET_LENGTH)
TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD = 'this-is-not-a-secure-password'
INSECURE_DEVELOPMENT_PASSWORD = 'this-is-not-a-secure-password'
DEFAULT_SIMULATION_REGISTRY_FILEPATH = os.path.join(DEFAULT_CONFIG_ROOT, 'simulated_registry.json')
MAX_TEST_SEEDER_ENTRIES = 20
MOCK_IP_ADDRESS = '0.0.0.0'
MOCK_IP_ADDRESS_2 = '10.10.10.10'
MOCK_URSULA_DB_FILEPATH = ':memory:'
MOCK_CUSTOM_INSTALLATION_PATH = '/tmp/nucypher-tmp-test-custom'

7
nucypher/utilities/sandbox/middleware.py
View File

@ -22,7 +22,7 @@ from bytestring_splitter import VariableLengthBytestring
from nucypher.characters.lawful import Ursula
from nucypher.crypto.kits import RevocationKit
from nucypher.network.middleware import RestMiddleware
from nucypher.utilities.sandbox.constants import TEST_KNOWN_URSULAS_CACHE
from nucypher.utilities.sandbox.constants import MOCK_KNOWN_URSULAS_CACHE
class MockRestMiddleware(RestMiddleware):
@ -47,13 +47,12 @@ class MockRestMiddleware(RestMiddleware):
def _get_ursula_by_port(self, port):
try:
return TEST_KNOWN_URSULAS_CACHE[port]
return MOCK_KNOWN_URSULAS_CACHE[port]
except KeyError:
raise RuntimeError(
"Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port))
def get_certificate(self, host, port, timeout=3, retry_attempts: int = 3,
retry_rate: int = 2, ):
def get_certificate(self, host, port, timeout=3, retry_attempts: int = 3, retry_rate: int = 2, current_attempt: int = 0):
ursula = self._get_ursula_by_port(port)
return ursula.certificate

164
nucypher/utilities/sandbox/ursula.py
View File

@ -14,45 +14,38 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
import random
import sys
import maya
import time
from os import linesep
import click
from eth_utils import to_checksum_address
from twisted.internet import reactor
from twisted.protocols.basic import LineReceiver
from typing import Set, Union
from typing import Union, Set
from nucypher.blockchain.eth import constants
from nucypher.blockchain.eth.constants import MIN_ALLOWED_LOCKED, MIN_LOCKED_PERIODS, MAX_MINTING_PERIODS
from nucypher.characters.lawful import Ursula
from nucypher.config.characters import UrsulaConfiguration
from nucypher.config.constants import SEEDNODES
from nucypher.crypto.api import secure_random
from nucypher.utilities.sandbox.constants import (DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
TEST_URSULA_STARTING_PORT,
TEST_KNOWN_URSULAS_CACHE)
from nucypher.utilities.sandbox.constants import (
MOCK_KNOWN_URSULAS_CACHE,
MOCK_URSULA_STARTING_PORT,
NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
MOCK_URSULA_DB_FILEPATH)
def make_federated_ursulas(ursula_config: UrsulaConfiguration,
quantity: int = DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
quantity: int = NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
know_each_other: bool = True,
**ursula_overrides) -> Set[Ursula]:
if not TEST_KNOWN_URSULAS_CACHE:
starting_port = TEST_URSULA_STARTING_PORT
if not MOCK_KNOWN_URSULAS_CACHE:
starting_port = MOCK_URSULA_STARTING_PORT
else:
starting_port = max(TEST_KNOWN_URSULAS_CACHE.keys()) + 1
starting_port = max(MOCK_KNOWN_URSULAS_CACHE.keys()) + 1
federated_ursulas = set()
for port in range(starting_port, starting_port+quantity):
ursula = ursula_config.produce(rest_port=port + 100,
db_name="test-{}".format(port),
db_filepath=MOCK_URSULA_DB_FILEPATH,
**ursula_overrides)
federated_ursulas.add(ursula)
@ -60,7 +53,7 @@ def make_federated_ursulas(ursula_config: UrsulaConfiguration,
# Store this Ursula in our global testing cache.
port = ursula.rest_information()[0].port
TEST_KNOWN_URSULAS_CACHE[port] = ursula
MOCK_KNOWN_URSULAS_CACHE[port] = ursula
if know_each_other:
@ -82,25 +75,25 @@ def make_decentralized_ursulas(ursula_config: UrsulaConfiguration,
if isinstance(ether_addresses, int):
ether_addresses = [to_checksum_address(secure_random(20)) for _ in range(ether_addresses)]
if not TEST_KNOWN_URSULAS_CACHE:
starting_port = TEST_URSULA_STARTING_PORT
if not MOCK_KNOWN_URSULAS_CACHE:
starting_port = MOCK_URSULA_STARTING_PORT
else:
starting_port = max(TEST_KNOWN_URSULAS_CACHE.keys()) + 1
starting_port = max(MOCK_KNOWN_URSULAS_CACHE.keys()) + 1
ursulas = set()
for port, checksum_address in enumerate(ether_addresses, start=starting_port):
ursula = ursula_config.produce(checksum_address=checksum_address,
db_name="test-{}".format(port),
ursula = ursula_config.produce(checksum_public_address=checksum_address,
db_filepath=MOCK_URSULA_DB_FILEPATH,
rest_port=port + 100,
**ursula_overrides)
if stake is True:
min_stake, balance = int(constants.MIN_ALLOWED_LOCKED), ursula.token_balance
min_stake, balance = MIN_ALLOWED_LOCKED, ursula.token_balance
amount = random.randint(min_stake, balance)
# for a random lock duration
min_locktime, max_locktime = int(constants.MIN_LOCKED_PERIODS), int(constants.MAX_MINTING_PERIODS)
min_locktime, max_locktime = MIN_LOCKED_PERIODS, MAX_MINTING_PERIODS
periods = random.randint(min_locktime, max_locktime)
ursula.initialize_stake(amount=amount, lock_periods=periods)
@ -108,7 +101,7 @@ def make_decentralized_ursulas(ursula_config: UrsulaConfiguration,
ursulas.add(ursula)
# Store this Ursula in our global cache.
port = ursula.rest_information()[0].port
TEST_KNOWN_URSULAS_CACHE[port] = ursula
MOCK_KNOWN_URSULAS_CACHE[port] = ursula
if know_each_other:
@ -119,116 +112,3 @@ def make_decentralized_ursulas(ursula_config: UrsulaConfiguration,
return ursulas
class UrsulaCommandProtocol(LineReceiver):
delimiter = linesep.encode("ascii")
encoding = 'utf-8'
width = 80
height = 24
commands = (
'status',
'stop',
)
def __init__(self, ursula):
self.ursula = ursula
self.start_time = maya.now()
super().__init__()
def _paint_known_nodes(self):
# Gather Data
known_nodes = self.ursula.known_nodes
known_certificate_files = os.listdir(self.ursula.known_certificates_dir)
number_of_known_nodes = len(known_nodes)
seen_nodes = len(known_certificate_files)
# Operating Mode
federated_only = self.ursula.federated_only
if federated_only:
click.secho("Configured in Federated Only mode", fg='green')
# Heading
label = "Known Nodes (connected {} / seen {})".format(number_of_known_nodes, seen_nodes)
heading = '\n' + label + " " * (45 - len(label)) + "Last Seen "
click.secho(heading, bold=True, nl=False)
# Legend
color_index = {
'self': 'yellow',
'known': 'white',
'seednode': 'blue'
}
for node_type, color in color_index.items():
click.secho('{0:<6} | '.format(node_type), fg=color, nl=False)
click.echo('\n')
seednode_addresses = list(bn.checksum_address for bn in SEEDNODES)
for address, node in known_nodes.items():
row_template = "{} | {} | {} | {} | {}"
node_type = 'known'
if node.checksum_public_address == self.ursula.checksum_public_address:
node_type = 'self'
row_template += ' ({})'.format(node_type)
elif node.checksum_public_address in seednode_addresses:
node_type = 'seednode'
row_template += ' ({})'.format(node_type)
click.secho(row_template.format(node.checksum_public_address,
node.rest_url().ljust(20), # TODO: Maybe put this 20 somewhere
node.nickname.ljust(50),
node.timestamp,
node.last_seen,
), fg=color_index[node_type])
def paintStatus(self):
stats = ['Ursula {}'.format(self.ursula.checksum_public_address),
'-'*50,
'Uptime: {}'.format(maya.now() - self.start_time), # TODO
'Learning Round: {}'.format(self.ursula._learning_round),
'Operating Mode: {}'.format('Federated' if self.ursula.federated_only else 'Decentralized'), # TODO
'Rest Interface {}'.format(self.ursula.rest_url()),
'Node Storage Type {}:'.format(self.ursula.node_storage._name.capitalize()),
'Known Nodes: {}'.format(len(self.ursula.known_nodes)),
'Work Orders: {}'.format(len(self.ursula._work_orders))]
if self.ursula._current_teacher_node:
teacher = 'Current Teacher: {}: ({})'.format(self.ursula._current_teacher_node,
self.ursula._current_teacher_node.rest_url())
stats.append(teacher)
click.echo('\n'+'\n'.join(stats))
def connectionMade(self):
message = '\nConnected to node console {}@{}'.format(self.ursula.checksum_public_address,
self.ursula.rest_url())
click.secho(message, fg='yellow')
click.secho("Type 'help' or '?' for help")
self.transport.write(b'Ursula >>> ')
def lineReceived(self, line):
line = line.decode(encoding=self.encoding).strip().lower()
commands = {
'known_nodes': self._paint_known_nodes,
'status': self.paintStatus,
'stop': reactor.stop,
'cycle_teacher': self.ursula.cycle_teacher_node
}
try:
commands[line]()
except KeyError:
click.secho("Invalid input. Options are {}".format(', '.join(commands)))
self.transport.write(b'Ursula >>> ')
def terminalSize(self, width, height):
self.width = width
self.height = height
self.terminal.eraseDisplay()
self._redraw()

5
setup.cfg Normal file
View File

@ -0,0 +1,5 @@
[aliases]
test=pytest
[tool:pytest]
python_files = tests/

10
setup.py
View File

@ -99,7 +99,7 @@ BENCHMARKS_REQUIRE = [
'pytest-benchmark'
]
EXTRAS_REQUIRE = {'testing': TESTS_REQUIRE,
EXTRAS_REQUIRE = {'test': TESTS_REQUIRE,
'deployment': DEPLOY_REQUIRES,
'docs': DOCS_REQUIRE,
'benchmark': BENCHMARKS_REQUIRE}
@ -113,6 +113,8 @@ setup(name=ABOUT['__title__'],
license=ABOUT['__license__'],
long_description=long_description,
setup_requires=['pytest-runner'], # required for setup.py test
tests_require=TESTS_REQUIRE,
install_requires=INSTALL_REQUIRES,
extras_require=EXTRAS_REQUIRE,
@ -127,7 +129,11 @@ setup(name=ABOUT['__title__'],
'blockchain/eth/sol/source/zepellin/token/*']},
include_package_data=True,
entry_points={'console_scripts': ['{0}={0}.cli:cli'.format(PACKAGE_NAME)]},
# Entry Points
entry_points={'console_scripts': [
'{0} = {0}.cli.main:nucypher_cli'.format(PACKAGE_NAME),
'{0}-deploy = {0}.cli.deploy:deploy'.format(PACKAGE_NAME),
]},
cmdclass={'verify': VerifyVersionCommand},
classifiers=[

31
tests/blockchain/eth/contracts/main/test_seeder.py → tests/blockchain/eth/contracts/base/seeder/test_seeder.py
View File

@ -14,34 +14,25 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
# import pytest
import pytest
from eth_tester.exceptions import TransactionFailed
TEST_MAX_SEEDS = 20
#
# def test_seeder(testerchain):
# origin, *everyone_else = testerchain.interface.w3.eth.accounts
# deployer = SeederDeployer(deployer_address=origin)
#
# agent = deployer.make_agent()
# direct_agent = SeederAgent()
#
# assert agent == direct_agent
#
from nucypher.blockchain.eth.constants import NULL_ADDRESS
from nucypher.utilities.sandbox.constants import MOCK_IP_ADDRESS, MOCK_IP_ADDRESS_2, MAX_TEST_SEEDER_ENTRIES, \
MOCK_URSULA_STARTING_PORT
@pytest.mark.slow()
def test_seeder(testerchain):
origin, seed_address, another_seed_address, *everyone_else = testerchain.interface.w3.eth.accounts
seed = ('0.0.0.0', 5757)
another_seed = ('10.10.10.10', 9151)
seed = (MOCK_IP_ADDRESS, MOCK_URSULA_STARTING_PORT)
another_seed = (MOCK_IP_ADDRESS_2, MOCK_URSULA_STARTING_PORT + 1)
contract, _txhash = testerchain.interface.deploy_contract('Seeder', TEST_MAX_SEEDS)
contract, _txhash = testerchain.interface.deploy_contract('Seeder', MAX_TEST_SEEDER_ENTRIES)
assert contract.functions.getSeedArrayLength().call() == TEST_MAX_SEEDS
assert contract.functions.getSeedArrayLength().call() == MAX_TEST_SEEDER_ENTRIES
assert contract.functions.owner().call() == origin
with pytest.raises((TransactionFailed, ValueError)):
@ -55,13 +46,13 @@ def test_seeder(testerchain):
testerchain.wait_for_receipt(txhash)
assert contract.functions.seeds(seed_address).call() == [*seed]
assert contract.functions.seedArray(0).call() == seed_address
assert contract.functions.seedArray(1).call() == "0x" + "0" * 40
assert contract.functions.seedArray(1).call() == NULL_ADDRESS
txhash = contract.functions.enroll(another_seed_address, *another_seed).transact({'from': origin})
testerchain.wait_for_receipt(txhash)
assert contract.functions.seeds(another_seed_address).call() == [*another_seed]
assert contract.functions.seedArray(0).call() == seed_address
assert contract.functions.seedArray(1).call() == another_seed_address
assert contract.functions.seedArray(2).call() == "0x" + "0" * 40
assert contract.functions.seedArray(2).call() == NULL_ADDRESS
txhash = contract.functions.refresh(*another_seed).transact({'from': seed_address})
testerchain.wait_for_receipt(txhash)

0
tests/blockchain/eth/contracts/main/test_intercontract_integration.py → tests/blockchain/eth/contracts/integration/test_intercontract_integration.py
View File

4
tests/blockchain/eth/entities/actors/test_miner.py
View File

@ -14,6 +14,8 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import maya
import pytest
@ -100,7 +102,7 @@ def test_miner_collects_staking_reward(testerchain, miner, three_agents):
initial_balance = miner.token_balance
assert token_agent.get_balance(miner.checksum_public_address) == initial_balance
miner.initialize_stake(amount=int(constants.MIN_ALLOWED_LOCKED), # Lock the minimum amount of tokens
miner.initialize_stake(amount=int(constants.MIN_ALLOWED_LOCKED), # Lock the minimum amount of tokens
lock_periods=int(constants.MIN_LOCKED_PERIODS)) # ... for the fewest number of periods
# ...wait out the lock period...

58
tests/characters/test_alice_can_grant_and_revoke.py
View File

@ -16,27 +16,24 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
import datetime
import maya
import os
import pytest
from apistar.test import TestClient
from constant_sorrow import constants
from nucypher.characters.lawful import Bob, Ursula
from nucypher.characters.lawful import Bob
from nucypher.config.characters import AliceConfiguration
from nucypher.config.storages import LocalFileBasedNodeStorage
from nucypher.crypto.api import keccak_digest
from nucypher.crypto.kits import RevocationKit
from nucypher.crypto.powers import SigningPower, DelegatingPower, EncryptingPower
from nucypher.crypto.powers import SigningPower, EncryptingPower
from nucypher.policy.models import Revocation
from nucypher.utilities.sandbox.constants import TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
from nucypher.utilities.sandbox.constants import INSECURE_DEVELOPMENT_PASSWORD
from nucypher.utilities.sandbox.middleware import MockRestMiddleware
from nucypher.utilities.sandbox.policy import MockPolicyCreation
from umbral.fragments import KFrag
@pytest.mark.skip(reason="to be implemented")
@pytest.mark.skip(reason="to be implemented") # TODO
@pytest.mark.usefixtures('blockchain_ursulas')
def test_mocked_decentralized_grant(blockchain_alice, blockchain_bob, three_agents):
@ -135,33 +132,26 @@ def test_revocation(federated_alice, federated_bob):
def test_alices_powers_are_persistent(federated_ursulas, tmpdir):
passphrase = TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
# Let's create an Alice from a Configuration.
# This requires creating a local storage for her first.
node_storage = LocalFileBasedNodeStorage(
federated_only=True,
character_class=Ursula, # Alice needs to store some info about Ursula
known_metadata_dir=os.path.join(tmpdir, "known_metadata"),
)
# Create a non-learning AliceConfiguration
alice_config = AliceConfiguration(
config_root=os.path.join(tmpdir, "config_root"),
node_storage=node_storage,
auto_initialize=True,
auto_generate_keys=True,
passphrase=passphrase,
is_me=True,
config_root=os.path.join(tmpdir, 'nucypher-custom-alice-config'),
network_middleware=MockRestMiddleware(),
known_nodes=federated_ursulas,
start_learning_now=False,
federated_only=True,
save_metadata=False,
load_metadata=False
)
alice = alice_config(passphrase=passphrase)
reload_metadata=False)
# We will save Alice's config to a file for later use
# Generate keys and write them the disk
alice_config.initialize(password=INSECURE_DEVELOPMENT_PASSWORD)
# Unlock Alice's keyring
alice_config.keyring.unlock(password=INSECURE_DEVELOPMENT_PASSWORD)
# Produce an Alice
alice = alice_config() # or alice_config.produce()
# Save Alice's node configuration file to disk for later use
alice_config_file = alice_config.to_configuration_file()
# Let's save Alice's public keys too to check they are correctly restored later
@ -181,8 +171,7 @@ def test_alices_powers_are_persistent(federated_ursulas, tmpdir):
bob = Bob(federated_only=True,
start_learning_now=False,
network_middleware=MockRestMiddleware(),
)
network_middleware=MockRestMiddleware())
bob_policy = alice.grant(bob, label, m=m, n=n, expiration=policy_end_datetime)
@ -208,7 +197,9 @@ def test_alices_powers_are_persistent(federated_ursulas, tmpdir):
start_learning_now=False,
)
new_alice = new_alice_config(passphrase=passphrase)
# Alice unlocks her restored keyring from disk
new_alice_config.keyring.unlock(password=INSECURE_DEVELOPMENT_PASSWORD)
new_alice = new_alice_config()
# First, we check that her public keys are correctly restored
assert alices_verifying_key == new_alice.public_keys(SigningPower)
@ -217,8 +208,7 @@ def test_alices_powers_are_persistent(federated_ursulas, tmpdir):
# Bob's eldest brother, Roberto, appears too
roberto = Bob(federated_only=True,
start_learning_now=False,
network_middleware=MockRestMiddleware(),
)
network_middleware=MockRestMiddleware())
# Alice creates a new policy for Roberto. Note how all the parameters
# except for the label (i.e., recipient, m, n, policy_end) are different

1
tests/characters/test_bob_handles_frags.py
View File

@ -80,7 +80,6 @@ def test_bob_can_follow_treasure_map_even_if_he_only_knows_of_one_node(enacted_f
from nucypher.characters.lawful import Bob
bob = Bob(network_middleware=MockRestMiddleware(),
known_certificates_dir=certificates_tempdir,
start_learning_now=False,
abort_on_learning_error=True,
federated_only=True)

5
tests/characters/test_bob_joins_policy_and_retrieves.py
View File

@ -6,7 +6,7 @@ import pytest
from nucypher.utilities.sandbox.middleware import MockRestMiddleware
from nucypher.characters.lawful import Bob, Ursula
from nucypher.data_sources import DataSource
from nucypher.utilities.sandbox.constants import DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
from nucypher.utilities.sandbox.constants import NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
from nucypher.keystore.keypairs import SigningKeypair
@ -52,7 +52,6 @@ def test_bob_joins_policy_and_retrieves(federated_alice,
bob = Bob(federated_only=True,
start_learning_now=True,
network_middleware=MockRestMiddleware(),
known_certificates_dir=certificates_tempdir,
abort_on_learning_error=True,
known_nodes=a_couple_of_ursulas,
)
@ -62,7 +61,7 @@ def test_bob_joins_policy_and_retrieves(federated_alice,
# Alice creates a policy granting access to Bob
# Just for fun, let's assume she distributes KFrags among Ursulas unknown to Bob
n = DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK - 2
n = NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK - 2
label = b'label://' + os.urandom(32)
contract_end_datetime = maya.now() + datetime.timedelta(days=5)
policy = federated_alice.grant(bob=bob,

2
tests/characters/test_crypto_characters_and_their_powers.py
View File

@ -99,7 +99,7 @@ def test_character_blockchain_power(testerchain):
sig_privkey = testerchain.interface.providers[0].ethereum_tester.backend._key_lookup[eth_utils.to_canonical_address(eth_address)]
sig_pubkey = sig_privkey.public_key
signer = Character(is_me=True, checksum_address=eth_address)
signer = Character(is_me=True, checksum_public_address=eth_address)
signer._crypto_power.consume_power_up(BlockchainPower(testerchain, eth_address))
# Due to testing backend, the account is already unlocked.

85
tests/characters/test_ursula_prepares_to_act_as_mining_node.py
View File

@ -14,15 +14,21 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
import pytest
from nucypher.characters.lawful import Ursula
from nucypher.characters.unlawful import Vladimir
from nucypher.crypto.powers import SigningPower, CryptoPower
from nucypher.utilities.sandbox.constants import INSECURE_DEVELOPMENT_PASSWORD
from nucypher.utilities.sandbox.middleware import MockRestMiddleware
from nucypher.utilities.sandbox.ursula import make_federated_ursulas
@pytest.mark.skip("To be implemented...?")
@pytest.mark.skip("To be implemented.")
def test_federated_ursula_substantiates_stamp():
assert False
@ -47,3 +53,80 @@ def test_new_federated_ursula_announces_herself(ursula_federated_test_config):
assert ursula_with_a_mouse in ursula_in_a_house.known_nodes
assert ursula_in_a_house in ursula_with_a_mouse.known_nodes
def test_blockchain_ursula_substantiates_stamp(blockchain_ursulas):
first_ursula = list(blockchain_ursulas)[0]
signature_as_bytes = first_ursula._evidence_of_decentralized_identity
signature = EthSignature(signature_bytes=signature_as_bytes)
proper_public_key_for_first_ursula = signature.recover_public_key_from_msg(bytes(first_ursula.stamp))
proper_address_for_first_ursula = proper_public_key_for_first_ursula.to_checksum_address()
assert proper_address_for_first_ursula == first_ursula.checksum_public_address
# This method is a shortcut for the above.
assert first_ursula._stamp_has_valid_wallet_signature
def test_blockchain_ursula_verifies_stamp(blockchain_ursulas):
first_ursula = list(blockchain_ursulas)[0]
# This Ursula does not yet have a verified stamp
first_ursula.verified_stamp = False
first_ursula.stamp_is_valid()
# ...but now it's verified.
assert first_ursula.verified_stamp
def test_vladimir_cannot_verify_interface_with_ursulas_signing_key(blockchain_ursulas):
his_target = list(blockchain_ursulas)[4]
# Vladimir has his own ether address; he hopes to publish it along with Ursula's details
# so that Alice (or whomever) pays him instead of Ursula, even though Ursula is providing the service.
# He finds a target and verifies that its interface is valid.
assert his_target.interface_is_valid()
# Now Vladimir imitates Ursula - copying her public keys and interface info, but inserting his ether address.
vladimir = Vladimir.from_target_ursula(his_target, claim_signing_key=True)
# Vladimir can substantiate the stamp using his own ether address...
vladimir.substantiate_stamp(password=INSECURE_DEVELOPMENT_PASSWORD)
vladimir.stamp_is_valid()
# Now, even though his public signing key matches Ursulas...
assert vladimir.stamp == his_target.stamp
# ...he is unable to pretend that his interface is valid
# because the interface validity check contains the canonical public address as part of its message.
with pytest.raises(vladimir.InvalidNode):
vladimir.interface_is_valid()
# Consequently, the metadata as a whole is also invalid.
with pytest.raises(vladimir.InvalidNode):
vladimir.validate_metadata()
def test_vladimir_uses_his_own_signing_key(blockchain_alice, blockchain_ursulas):
"""
Similar to the attack above, but this time Vladimir makes his own interface signature
using his own signing key, which he claims is Ursula's.
"""
his_target = list(blockchain_ursulas)[4]
fraduluent_keys = CryptoPower(power_ups=Ursula._default_crypto_powerups) # TODO: Why is this unused?
vladimir = Vladimir.from_target_ursula(target_ursula=his_target)
message = vladimir._signable_interface_info_message()
signature = vladimir._crypto_power.power_ups(SigningPower).sign(vladimir.timestamp_bytes() + message)
vladimir._interface_signature_object = signature
vladimir.substantiate_stamp(password=INSECURE_DEVELOPMENT_PASSWORD)
# With this slightly more sophisticated attack, his metadata does appear valid.
vladimir.validate_metadata()
# However, the actual handshake proves him wrong.
with pytest.raises(vladimir.InvalidNode):
vladimir.verify_node(blockchain_alice.network_middleware)

41
tests/cli/commands/test_help.py Normal file
View File

@ -0,0 +1,41 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from nucypher.cli.deploy import deploy
from nucypher.cli.main import nucypher_cli
def test_nucypher_help_message(click_runner):
help_args = ('--help', )
result = click_runner.invoke(nucypher_cli, help_args, catch_exceptions=False)
assert result.exit_code == 0
assert '[OPTIONS] COMMAND [ARGS]' in result.output, 'Missing or invalid help text was produced.'
def test_nucypher_ursula_help_message(click_runner):
help_args = ('ursula', '--help')
result = click_runner.invoke(nucypher_cli, help_args, catch_exceptions=False)
assert result.exit_code == 0
assert 'ursula [OPTIONS] ACTION' in result.output, 'Missing or invalid help text was produced.'
def test_nucypher_deploy_help_message(click_runner):
help_args = ('--help', )
result = click_runner.invoke(deploy, help_args, catch_exceptions=False)
assert result.exit_code == 0
assert 'deploy [OPTIONS] ACTION' in result.output, 'Missing or invalid help text was produced.'

31
tests/cli/test_run_ursula.py → tests/cli/commands/test_run_ursula.py
View File

@ -14,33 +14,30 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import time
import pytest
import pytest_twisted
from click.testing import CliRunner
import pytest_twisted as pt
import time
from twisted.internet import threads
from twisted.internet.error import CannotListenError
from nucypher.cli import cli
from nucypher.characters.base import Learner
from nucypher.cli.main import nucypher_cli
from nucypher.utilities.sandbox.constants import INSECURE_DEVELOPMENT_PASSWORD, MOCK_URSULA_STARTING_PORT
from nucypher.utilities.sandbox.constants import TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
from nucypher.utilities.sandbox.ursula import UrsulaCommandProtocol
@pytest.mark.skip()
@pytest_twisted.inlineCallbacks
def test_run_lone_federated_default_ursula():
@pytest.mark.skip('Results in exception "ReactorAlreadyRunning"')
@pt.inlineCallbacks
def test_run_lone_federated_default_development_ursula(click_runner):
args = ('ursula', 'run', '--rest-port', MOCK_URSULA_STARTING_PORT, '--dev')
args = ['--dev',
'--federated-only',
'ursula', 'run',
'--rest-port', '9999', # TODO: use different port to avoid premature ConnectionError with many test runs?
'--no-reactor'
]
runner = CliRunner()
result = yield threads.deferToThread(runner.invoke, cli, args, catch_exceptions=False, input=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD+'\n')
result = yield threads.deferToThread(click_runner.invoke,
nucypher_cli, args,
catch_exceptions=False,
input=INSECURE_DEVELOPMENT_PASSWORD + '\n')
alone = "WARNING - Can't learn right now: Need some nodes to start learning from."
time.sleep(Learner._SHORT_LEARNING_DELAY)
@ -49,4 +46,4 @@ def test_run_lone_federated_default_ursula():
# Cannot start another Ursula on the same REST port
with pytest.raises(CannotListenError):
_result = runner.invoke(cli, args, catch_exceptions=False, input=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD)
_result = click_runner.invoke(nucypher_cli, args, catch_exceptions=False, input=INSECURE_DEVELOPMENT_PASSWORD)

25
tests/cli/test_stake.py → tests/cli/commands/test_stake.py
View File

@ -14,25 +14,24 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import pytest
from click.testing import CliRunner
from nucypher.cli import cli
from nucypher.cli.main import nucypher_cli
@pytest.mark.skip
def test_stake_init():
runner = CliRunner()
result = runner.invoke(cli, ['stake', 'init'], catch_exceptions=False)
@pytest.mark.skip("To be implemented") # TODO
def test_stake_init(click_runner):
result = click_runner.invoke(nucypher_cli, ['stake', 'init'], catch_exceptions=False)
@pytest.mark.skip
def test_stake_info():
runner = CliRunner()
result = runner.invoke(cli, ['stake', 'info'], catch_exceptions=False)
@pytest.mark.skip("To be implemented") # TODO
def test_stake_info(click_runner):
result = click_runner.invoke(nucypher_cli, ['stake', 'info'], catch_exceptions=False)
@pytest.mark.skip
def test_stake_confirm():
runner = CliRunner()
result = runner.invoke(cli, ['stake', 'confirm-activity'], catch_exceptions=False)
@pytest.mark.skip("To be implemented") # TODO
def test_stake_confirm(click_runner):
result = click_runner.invoke(nucypher_cli, ['stake', 'confirm-activity'], catch_exceptions=False)

64
tests/cli/commands/test_status.py Normal file
View File

@ -0,0 +1,64 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
from nucypher.cli.main import nucypher_cli
from nucypher.config.characters import UrsulaConfiguration
from nucypher.utilities.sandbox.constants import INSECURE_DEVELOPMENT_PASSWORD, MOCK_CUSTOM_INSTALLATION_PATH, \
MOCK_IP_ADDRESS_2
def test_initialize_configuration_files_and_directories(custom_filepath, click_runner):
init_args = ('ursula', 'init', '--config-root', custom_filepath)
# Use a custom local filepath for configuration
user_input = '{ip}\n{password}\n{password}\n'.format(password=INSECURE_DEVELOPMENT_PASSWORD, ip=MOCK_IP_ADDRESS_2)
result = click_runner.invoke(nucypher_cli, init_args, input=user_input, catch_exceptions=False)
assert result.exit_code == 0
# CLI Output
assert MOCK_CUSTOM_INSTALLATION_PATH in result.output, "Configuration not in system temporary directory"
assert "nucypher ursula run" in result.output, 'Help message is missing suggested command'
# Files and Directories
assert os.path.isdir(custom_filepath), 'Configuration file does not exist'
assert os.path.isdir(os.path.join(custom_filepath, 'keyring')), 'Keyring does not exist'
assert os.path.isdir(os.path.join(custom_filepath, 'known_nodes')), 'known_nodes directory does not exist'
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
# Auth
assert 'Enter keyring password:' in result.output, 'WARNING: User was not prompted for password'
assert 'Repeat for confirmation:' in result.output, 'User was not prompted to confirm password'
def test_empty_federated_status(click_runner, custom_filepath):
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
status_args = ('status', '--config-file', custom_config_filepath)
result = click_runner.invoke(nucypher_cli, status_args, catch_exceptions=False)
assert result.exit_code == 0
assert 'Federated Only' in result.output
heading = 'Known Nodes (connected 0 / seen 0)'
assert heading in result.output
assert 'password' not in result.output

225
tests/cli/commands/test_ursula.py Normal file
View File

@ -0,0 +1,225 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import json
import os
from json import JSONDecodeError
import pytest
from nucypher.cli.main import nucypher_cli
from nucypher.config.characters import UrsulaConfiguration
from nucypher.config.constants import APP_DIR, DEFAULT_CONFIG_ROOT
from nucypher.utilities.sandbox.constants import (
INSECURE_DEVELOPMENT_PASSWORD,
MOCK_CUSTOM_INSTALLATION_PATH,
MOCK_IP_ADDRESS,
MOCK_URSULA_STARTING_PORT
)
def test_initialize_ursula_defaults(click_runner, mocker):
# Mock out filesystem writes
mocker.patch.object(UrsulaConfiguration, 'initialize', autospec=True)
mocker.patch.object(UrsulaConfiguration, 'to_configuration_file', autospec=True)
# Use default ursula init args
init_args = ('ursula', 'init')
user_input = '{ip}\n{password}\n{password}\n'.format(password=INSECURE_DEVELOPMENT_PASSWORD, ip=MOCK_IP_ADDRESS)
result = click_runner.invoke(nucypher_cli, init_args, input=user_input, catch_exceptions=False)
assert result.exit_code == 0
# REST Host
assert 'Enter Ursula\'s public-facing IPv4 address' in result.output
# Auth
assert 'Enter keyring password:' in result.output, 'WARNING: User was not prompted for password'
assert 'Repeat for confirmation:' in result.output, 'User was not prompted to confirm password'
def test_initialize_custom_configuration_root(custom_filepath, click_runner):
# Use a custom local filepath for configuration
init_args = ('ursula', 'init',
'--config-root', custom_filepath,
'--rest-host', MOCK_IP_ADDRESS,
'--rest-port', MOCK_URSULA_STARTING_PORT)
user_input = '{password}\n{password}'.format(password=INSECURE_DEVELOPMENT_PASSWORD, ip=MOCK_IP_ADDRESS)
result = click_runner.invoke(nucypher_cli, init_args, input=user_input, catch_exceptions=False)
assert result.exit_code == 0
# CLI Output
assert MOCK_CUSTOM_INSTALLATION_PATH in result.output, "Configuration not in system temporary directory"
assert "nucypher ursula run" in result.output, 'Help message is missing suggested command'
assert 'IPv4' not in result.output
# Files and Directories
assert os.path.isdir(custom_filepath), 'Configuration file does not exist'
assert os.path.isdir(os.path.join(custom_filepath, 'keyring')), 'Keyring does not exist'
assert os.path.isdir(os.path.join(custom_filepath, 'known_nodes')), 'known_nodes directory does not exist'
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
# Auth
assert 'Enter keyring password:' in result.output, 'WARNING: User was not prompted for password'
assert 'Repeat for confirmation:' in result.output, 'User was not prompted to confirm password'
def test_configuration_file_contents(custom_filepath, nominal_configuration_fields):
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
# Check the contents of the configuration file
with open(custom_config_filepath, 'r') as config_file:
raw_contents = config_file.read()
try:
data = json.loads(raw_contents)
except JSONDecodeError:
raise pytest.fail(msg="Invalid JSON configuration file {}".format(custom_config_filepath))
for field in nominal_configuration_fields:
assert field in data, "Missing field '{}' from configuration file."
if any(keyword in field for keyword in ('path', 'dir')):
path = data[field]
user_data_dir = APP_DIR.user_data_dir
# assert os.path.exists(path), '{} does not exist'.format(path)
assert user_data_dir not in path, '{} includes default appdir path {}'.format(field, user_data_dir)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
def test_password_prompt(click_runner, custom_filepath):
# Ensure the configuration file still exists
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
view_args = ('ursula', 'view', '--config-file', custom_config_filepath)
user_input = '{}\n'.format(INSECURE_DEVELOPMENT_PASSWORD)
result = click_runner.invoke(nucypher_cli, view_args, input=user_input, catch_exceptions=False, env=dict())
assert 'password' in result.output, 'WARNING: User was not prompted for password'
assert result.exit_code == 0
envvars = {'NUCYPHER_KEYRING_PASSWORD': INSECURE_DEVELOPMENT_PASSWORD}
result = click_runner.invoke(nucypher_cli, view_args, input=user_input, catch_exceptions=False, env=envvars)
assert not 'password' in result.output, 'User was prompted for password'
assert result.exit_code == 0
def test_ursula_view_configuration(custom_filepath, click_runner, nominal_configuration_fields):
# Ensure the configuration file still exists
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
view_args = ('ursula', 'view', '--config-file', os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME))
# View the configuration
result = click_runner.invoke(nucypher_cli, view_args,
input='{}\n'.format(INSECURE_DEVELOPMENT_PASSWORD),
catch_exceptions=False)
# CLI Output
assert 'password' in result.output, 'WARNING: User was not prompted for password'
assert MOCK_CUSTOM_INSTALLATION_PATH in result.output
for field in nominal_configuration_fields:
assert field in result.output, "Missing field '{}' from configuration file."
# Make sure nothing crazy is happening...
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
@pytest.mark.skip("Results in ReactorAlreadyRunning") # TODO: Find a way to execute this test (contains reactor.run call)
def test_run_ursula(custom_filepath, click_runner):
# Ensure the configuration file still exists
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
# Run Ursula
run_args = ('ursula', 'run', '--config-file', custom_config_filepath)
result = click_runner.invoke(nucypher_cli, run_args,
input='{}\nY\n'.format(INSECURE_DEVELOPMENT_PASSWORD),
catch_exceptions=False)
# CLI Output
assert result.exit_code == 0
assert 'password' in result.output, 'WARNING: User was not prompted for password'
assert '? [y/N]:' in result.output, 'WARNING: User was to run Ursula'
assert '>>>' in result.output
def test_ursula_init_does_not_overrides_existing_files(custom_filepath, click_runner):
# Ensure the configuration file still exists
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
init_args = ('ursula', 'init', '--config-root', custom_filepath, '--rest-host', MOCK_IP_ADDRESS)
# Ensure that an existing configuration directory cannot be overridden
with pytest.raises(UrsulaConfiguration.ConfigurationError):
_bad_result = click_runner.invoke(nucypher_cli, init_args,
input='{}\n'.format(INSECURE_DEVELOPMENT_PASSWORD)*2,
catch_exceptions=False)
assert 'password' in _bad_result.output, 'WARNING: User was not prompted for password'
# Really we want to keep this file until its destroyed
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
def test_ursula_destroy_configuration(custom_filepath, click_runner):
preexisting_live_configuration = os.path.isdir(DEFAULT_CONFIG_ROOT)
preexisting_live_configuration_file = os.path.isfile(os.path.join(DEFAULT_CONFIG_ROOT, UrsulaConfiguration.CONFIG_FILENAME))
# Ensure the configuration file still exists
custom_config_filepath = os.path.join(custom_filepath, UrsulaConfiguration.CONFIG_FILENAME)
assert os.path.isfile(custom_config_filepath), 'Configuration file does not exist'
# Run the destroy command
destruction_args = ('ursula', 'destroy', '--config-file', custom_config_filepath)
result = click_runner.invoke(nucypher_cli, destruction_args,
input='{}\nY\n'.format(INSECURE_DEVELOPMENT_PASSWORD),
catch_exceptions=False)
# CLI Output
assert not os.path.isfile(custom_config_filepath), 'Configuration file still exists'
assert 'password' in result.output, 'WARNING: User was not prompted for password'
assert '? [y/N]:' in result.output, 'WARNING: User was not asked to destroy files'
assert custom_filepath in result.output, 'WARNING: Configuration path not in output. Deleting the wrong path?'
assert 'Deleted' in result.output, '"Deleted" not in output'
assert result.exit_code == 0, 'Destruction did not succeed'
# Ensure the files are deleted from the filesystem
assert not os.path.isfile(custom_config_filepath), 'Files still exist' # ... it's gone...
assert not os.path.isdir(custom_filepath), 'Nucypher files still exist' # it's all gone...
# If this test started off with a live configuration, ensure it still exists
if preexisting_live_configuration:
configuration_still_exists = os.path.isdir(DEFAULT_CONFIG_ROOT)
assert configuration_still_exists
if preexisting_live_configuration_file:
file_still_exists = os.path.isfile(os.path.join(DEFAULT_CONFIG_ROOT, UrsulaConfiguration.CONFIG_FILENAME))
assert file_still_exists, 'WARNING: Test command deleted live non-test files'

51
tests/cli/conftest.py Normal file
View File

@ -0,0 +1,51 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import contextlib
import shutil
import pytest
from click.testing import CliRunner
from nucypher.config.characters import UrsulaConfiguration
from nucypher.utilities.sandbox.constants import MOCK_CUSTOM_INSTALLATION_PATH
@pytest.fixture(scope='module')
def click_runner():
runner = CliRunner()
yield runner
@pytest.fixture(scope='module')
def nominal_configuration_fields():
config = UrsulaConfiguration(dev_mode=True)
config_fields = config.static_payload
del config_fields['is_me']
yield tuple(config_fields.keys())
del config
@pytest.fixture(scope='module')
def custom_filepath():
_custom_filepath = MOCK_CUSTOM_INSTALLATION_PATH
with contextlib.suppress(FileNotFoundError):
shutil.rmtree(_custom_filepath, ignore_errors=True)
try:
yield _custom_filepath
finally:
with contextlib.suppress(FileNotFoundError):
shutil.rmtree(_custom_filepath, ignore_errors=True)

90
tests/cli/protocol/test_ursula_command.py Normal file
View File

@ -0,0 +1,90 @@
import sys
from contextlib import contextmanager
import pytest
from io import StringIO
from nucypher.cli.main import NucypherClickConfig
from nucypher.cli.protocol import UrsulaCommandProtocol
# Disable click sentry and file logging
NucypherClickConfig.log_to_sentry = False
NucypherClickConfig.log_to_file = False
@contextmanager
def capture_output():
new_out, new_err = StringIO(), StringIO()
old_out, old_err = sys.stdout, sys.stderr
try:
sys.stdout, sys.stderr = new_out, new_err
yield sys.stdout, sys.stderr
finally:
sys.stdout, sys.stderr = old_out, old_err
@pytest.fixture(scope='module')
def ursula(federated_ursulas):
ursula = federated_ursulas.pop()
return ursula
@pytest.fixture(scope='module')
def protocol(ursula):
protocol = UrsulaCommandProtocol(ursula=ursula)
return protocol
def test_ursula_command_protocol_creation(ursula):
protocol = UrsulaCommandProtocol(ursula=ursula)
assert protocol.ursula == ursula
assert b'Ursula' in protocol.prompt
def test_ursula_command_help(protocol, ursula):
class FakeTransport:
"""This is a transport"""
mock_output = b''
@staticmethod
def write(data: bytes):
FakeTransport.mock_output += data
protocol.transport = FakeTransport
with capture_output() as (out, err):
protocol.lineReceived(line=b'bananas')
# Ensure all commands are in the help text
result = out.getvalue()
for command in protocol.commands:
assert command in result, '{} is missing from help text'.format(command)
# Blank lines are OK!
with capture_output() as (out, err):
protocol.lineReceived(line=b'')
assert protocol.prompt in FakeTransport.mock_output
def test_ursula_command_status(protocol, ursula):
with capture_output() as (out, err):
protocol.paintStatus()
result = out.getvalue()
assert ursula.checksum_public_address in result
assert '...' in result
assert 'Known Nodes' in result
def test_ursula_command_known_nodes(protocol, ursula):
with capture_output() as (out, err):
protocol.paintKnownNodes()
result = out.getvalue()
assert 'Known Nodes' in result
assert ursula.checksum_public_address not in result

62
tests/cli/test_accounts.py
View File

@ -1,62 +0,0 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import pytest
from click.testing import CliRunner
from nucypher.cli import cli
@pytest.mark.usefixtures("three_agents")
def test_list(testerchain):
runner = CliRunner()
account = testerchain.interface.w3.eth.accounts[0]
args = '--dev --federated-only --provider-uri tester://pyevm accounts list'.split()
result = runner.invoke(cli, args, catch_exceptions=False)
assert result.exit_code == 0
assert account in result.output
@pytest.mark.usefixtures("three_agents")
def test_balance(testerchain):
runner = CliRunner()
account = testerchain.interface.w3.eth.accounts[0]
args = '--dev --federated-only --provider-uri tester://pyevm accounts balance'.split()
result = runner.invoke(cli, args, catch_exceptions=False)
assert result.exit_code == 0
assert 'Tokens:' in result.output
assert 'ETH:' in result.output
assert account in result.output
@pytest.mark.usefixtures("three_agents")
def test_transfer_eth(testerchain):
runner = CliRunner()
account = testerchain.interface.w3.eth.accounts[1]
args = '--dev --federated-only --provider-uri tester://pyevm accounts transfer-eth'.split()
result = runner.invoke(cli, args, catch_exceptions=False, input=account+'\n100\nY\n')
assert result.exit_code == 0
@pytest.mark.usefixtures("three_agents")
def test_transfer_tokens(testerchain):
runner = CliRunner()
account = testerchain.interface.w3.eth.accounts[2]
args = '--dev --federated-only --provider-uri tester://pyevm accounts transfer-tokens'.split()
result = runner.invoke(cli, args, catch_exceptions=False, input=account+'\n100\nY\n')
assert result.exit_code == 0

109
tests/cli/test_config.py
View File

@ -1,109 +0,0 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import contextlib
import os
import pytest
import shutil
from click.testing import CliRunner
from nucypher.cli import cli
from nucypher.config.node import NodeConfiguration
from nucypher.utilities.sandbox.constants import TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
TEST_CUSTOM_INSTALLATION_PATH = '/tmp/nucypher-tmp-test-custom'
@pytest.fixture(scope='function')
def custom_filepath():
custom_filepath = TEST_CUSTOM_INSTALLATION_PATH
yield custom_filepath
with contextlib.suppress(FileNotFoundError):
shutil.rmtree(custom_filepath, ignore_errors=True)
@pytest.mark.skip()
def test_initialize_configuration_files_and_directories(custom_filepath):
runner = CliRunner()
# Use the system temporary storage area
args = ['--dev', '--federated-only', 'configure', 'install', '--ursula', '--force']
result = runner.invoke(cli, args,
input='{}\n{}'''.format(TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD),
catch_exceptions=False)
assert '/tmp' in result.output, "Configuration not in system temporary directory"
assert NodeConfiguration._NodeConfiguration__TEMP_CONFIGURATION_DIR_PREFIX in result.output
assert result.exit_code == 0
# Use a custom local filepath
args = ['--config-root', custom_filepath, '--federated-only', 'configure', 'install', '--ursula', '--force']
result = runner.invoke(cli, args,
input='{}\n{}'''.format(TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD),
catch_exceptions=False)
assert TEST_CUSTOM_INSTALLATION_PATH in result.output, "Configuration not in system temporary directory"
assert 'Created' in result.output
assert custom_filepath in result.output
assert "'nucypher ursula run'" in result.output
assert result.exit_code == 0
assert os.path.isdir(custom_filepath)
# Ensure that there are not pre-existing configuration files at config_root
_result = runner.invoke(cli, args,
input='{}\n{}'''.format(TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD),
catch_exceptions=False)
assert "There are existing configuration files" in _result.output
# Destroy / Uninstall
args = ['--config-root', custom_filepath, 'configure', 'destroy']
result = runner.invoke(cli, args, input='Y', catch_exceptions=False)
assert '[y/N]' in result.output
assert TEST_CUSTOM_INSTALLATION_PATH in result.output, "Configuration not in system temporary directory"
assert 'Deleted' in result.output
assert custom_filepath in result.output
assert result.exit_code == 0
assert not os.path.isdir(custom_filepath)
# # TODO: Integrate with run ursula
@pytest.mark.skip()
def test_validate_runtime_filepaths(custom_filepath):
runner = CliRunner()
args = ['--config-root', custom_filepath, 'configure', 'install', '--no-registry']
result = runner.invoke(cli, args, input='Y', catch_exceptions=False)
result = runner.invoke(cli, ['--config-root', custom_filepath,
'configure', 'validate',
'--filesystem',
'--no-registry'], catch_exceptions=False)
assert custom_filepath in result.output
assert 'Valid' in result.output
assert result.exit_code == 0
# Remove the known nodes dir to "corrupt" the tree
shutil.rmtree(os.path.join(custom_filepath, 'known_nodes'))
result = runner.invoke(cli, ['--config-root', custom_filepath,
'configure', 'validate',
'--filesystem',
'--no-registry'], catch_exceptions=False)
assert custom_filepath in result.output
assert 'Invalid' in result.output
assert result.exit_code == 0 # TODO: exit differently for invalidity?

8
tests/config/test_firstula_circumstances.py
View File

@ -14,6 +14,8 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from functools import partial
import pytest_twisted
@ -45,6 +47,7 @@ def test_get_cert_from_running_seed_node(ursula_federated_test_config):
ursula_config=ursula_federated_test_config,
quantity=1,
know_each_other=False)
firstula = lonely_ursula_maker().pop()
node_deployer = firstula.get_deployer()
@ -61,6 +64,11 @@ def test_get_cert_from_running_seed_node(ursula_federated_test_config):
def start_lonely_learning_loop():
any_other_ursula.start_learning_loop()
start = maya.now()
while firstula not in any_other_ursula.known_nodes:
passed = maya.now() - start
if passed.seconds > 2:
pytest.fail("Didn't find the seed node.")
any_other_ursula.block_until_specific_nodes_are_known(set([firstula.checksum_public_address]), timeout=2)
yield deferToThread(start_lonely_learning_loop)

18
tests/config/test_keyring.py
View File

@ -8,22 +8,22 @@ from nucypher.crypto.powers import DelegatingPower, EncryptingPower
@pytest.mark.skip("Redacted and refactored for sensitive info leakage")
def test_validate_passphrase():
# Passphrase too short
passphrase = 'x' * 5
def test_validate_password():
# Password too short
password = 'x' * 5
with pytest.raises(ValueError):
_keyring = NucypherKeyring.generate(passphrase=passphrase)
_keyring = NucypherKeyring.generate(password=password)
# Empty passphrase is provided
# Empty password is provided
with pytest.raises(ValueError):
_keyring = NucypherKeyring.generate(passphrase="")
_keyring = NucypherKeyring.generate(password="")
def test_generate_alice_keyring(tmpdir):
passphrase = 'x' * 16
password = 'x' * 16
keyring = NucypherKeyring.generate(
passphrase=passphrase,
password=password,
encrypting=True,
wallet=False,
tls=False,
@ -36,7 +36,7 @@ def test_generate_alice_keyring(tmpdir):
with pytest.raises(NucypherKeyring.KeyringLocked):
_enc_keypair = keyring.derive_crypto_power(EncryptingPower).keypair
keyring.unlock(passphrase)
keyring.unlock(password)
enc_keypair = keyring.derive_crypto_power(EncryptingPower).keypair
assert enc_pubkey == enc_keypair.pubkey

43
tests/config/test_storages.py
View File

@ -27,11 +27,11 @@ from moto import mock_s3
from nucypher.characters.lawful import Ursula
from nucypher.config.storages import (
S3NodeStorage,
InMemoryNodeStorage,
ForgetfulNodeStorage,
TemporaryFileBasedNodeStorage,
NodeStorage,
LocalFileBasedNodeStorage
NodeStorage
)
from nucypher.utilities.sandbox.constants import MOCK_URSULA_DB_FILEPATH
MOCK_S3_BUCKET_NAME = 'mock-seednodes'
S3_DOMAIN_NAME = 's3.amazonaws.com'
@ -41,26 +41,25 @@ class BaseTestNodeStorageBackends:
@pytest.fixture(scope='class')
def light_ursula(temp_dir_path):
db_name = 'ursula-{}.db'.format(10151)
db_filepath = 'ursula-{}.db'.format(10151)
try:
node = Ursula(rest_host='127.0.0.1',
rest_port=10151,
db_filepath=db_name,
db_name=db_name,
db_filepath=MOCK_URSULA_DB_FILEPATH,
federated_only=True)
yield node
finally:
with contextlib.suppress(Exception):
os.remove(db_name)
os.remove(db_filepath)
character_class = Ursula
federated_only = True
storage_backend = NotImplemented
def _read_and_write_to_storage(self, ursula, node_storage):
def _read_and_write_metadata(self, ursula, node_storage):
# Write Node
node_storage.save(node=ursula)
node_storage.store_node_metadata(node=ursula)
# Read Node
node_from_storage = node_storage.get(checksum_address=ursula.checksum_public_address,
@ -70,8 +69,8 @@ class BaseTestNodeStorageBackends:
# Save more nodes
all_known_nodes = set()
for port in range(10152, 10251):
node = Ursula(rest_host='127.0.0.1', rest_port=port, federated_only=True)
node_storage.save(node=node)
node = Ursula(rest_host='127.0.0.1', db_filepath=MOCK_URSULA_DB_FILEPATH, rest_port=port, federated_only=True)
node_storage.store_node_metadata(node=node)
all_known_nodes.add(node)
# Read all nodes from storage
@ -89,12 +88,12 @@ class BaseTestNodeStorageBackends:
return True
def _write_and_delete_nodes_in_storage(self, ursula, node_storage):
def _write_and_delete_metadata(self, ursula, node_storage):
# Write Node
node_storage.save(node=ursula)
node_storage.store_node_metadata(node=ursula)
# Delete Node
node_storage.remove(checksum_address=ursula.checksum_public_address)
node_storage.remove(checksum_address=ursula.checksum_public_address, certificate=False)
# Read Node
with pytest.raises(NodeStorage.UnknownNode):
@ -112,15 +111,15 @@ class BaseTestNodeStorageBackends:
#
def test_delete_node_in_storage(self, light_ursula):
assert self._write_and_delete_nodes_in_storage(ursula=light_ursula, node_storage=self.storage_backend)
assert self._write_and_delete_metadata(ursula=light_ursula, node_storage=self.storage_backend)
def test_read_and_write_to_storage(self, light_ursula):
assert self._read_and_write_to_storage(ursula=light_ursula, node_storage=self.storage_backend)
assert self._read_and_write_metadata(ursula=light_ursula, node_storage=self.storage_backend)
class TestInMemoryNodeStorage(BaseTestNodeStorageBackends):
storage_backend = InMemoryNodeStorage(character_class=BaseTestNodeStorageBackends.character_class,
federated_only=BaseTestNodeStorageBackends.federated_only)
storage_backend = ForgetfulNodeStorage(character_class=BaseTestNodeStorageBackends.character_class,
federated_only=BaseTestNodeStorageBackends.federated_only)
storage_backend.initialize()
@ -149,7 +148,7 @@ class TestS3NodeStorageDirect(BaseTestNodeStorageBackends):
@mock_s3
def test_generate_presigned_url(self, light_ursula):
s3_node_storage = self.s3_node_storage_factory()
s3_node_storage.save(node=light_ursula)
s3_node_storage.store_node_metadata(node=light_ursula)
presigned_url = s3_node_storage.generate_presigned_url(checksum_address=light_ursula.checksum_public_address)
assert S3_DOMAIN_NAME in presigned_url
@ -164,7 +163,7 @@ class TestS3NodeStorageDirect(BaseTestNodeStorageBackends):
s3_node_storage = self.s3_node_storage_factory()
# Write Node
s3_node_storage.save(node=light_ursula)
s3_node_storage.store_node_metadata(node=light_ursula)
# Read Node
node_from_storage = s3_node_storage.get(checksum_address=light_ursula.checksum_public_address,
@ -175,7 +174,7 @@ class TestS3NodeStorageDirect(BaseTestNodeStorageBackends):
all_known_nodes = set()
for port in range(10152, 10251):
node = Ursula(rest_host='127.0.0.1', rest_port=port, federated_only=True)
s3_node_storage.save(node=node)
s3_node_storage.store_node_metadata(node=node)
all_known_nodes.add(node)
# Read all nodes from storage
@ -198,7 +197,7 @@ class TestS3NodeStorageDirect(BaseTestNodeStorageBackends):
s3_node_storage = self.s3_node_storage_factory()
# Write Node
s3_node_storage.save(node=light_ursula)
s3_node_storage.store_node_metadata(node=light_ursula)
# Delete Node
s3_node_storage.remove(checksum_address=light_ursula.checksum_public_address)

21
tests/conftest.py
View File

@ -18,15 +18,27 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
import pytest
from twisted.logger import globalLogPublisher
from nucypher.cli.main import NucypherClickConfig
from nucypher.utilities.logging import SimpleObserver
#
from nucypher.cli import NucypherClickConfig
from nucypher.utilities.logging import SimpleObserver
# Logger Configuration
NucypherClickConfig.log_to_sentry = False
#
# Disable click sentry and file logging
NucypherClickConfig.log_to_sentry = False
NucypherClickConfig.log_to_file = False
#
# Pytest configuration
#
pytest_plugins = [
'tests.fixtures',
'tests.fixtures', # Includes external fixtures module
]
@ -47,3 +59,8 @@ def pytest_collection_modifyitems(config, items):
log_level_name = config.getoption("--log-level", "info", skip=True)
observer = SimpleObserver(log_level_name)
globalLogPublisher.addObserver(observer)
# Timber!
log_level_name = config.getoption("--log-level", "info", skip=True)
observer = SimpleObserver(log_level_name)
globalLogPublisher.addObserver(observer)

84
tests/fixtures.py
View File

@ -14,23 +14,25 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import contextlib
import glob
import os
import tempfile
import datetime
import os
import re
import shutil
import tempfile
import maya
import pytest
from constant_sorrow import constants
import re
import shutil
from sqlalchemy.engine import create_engine
from constant_sorrow.constants import NON_PAYMENT
from nucypher.blockchain.eth.constants import DISPATCHER_SECRET_LENGTH
from nucypher.blockchain.eth.deployers import PolicyManagerDeployer, NucypherTokenDeployer, MinerEscrowDeployer
from nucypher.blockchain.eth.interfaces import BlockchainDeployerInterface
from nucypher.blockchain.eth.registry import TemporaryEthereumContractRegistry, InMemoryEthereumContractRegistry
from nucypher.blockchain.eth.registry import InMemoryEthereumContractRegistry
from nucypher.blockchain.eth.sol.compile import SolidityCompiler
from nucypher.config.characters import UrsulaConfiguration, AliceConfiguration, BobConfiguration
from nucypher.config.constants import BASE_DIR
@ -40,9 +42,8 @@ from nucypher.keystore import keystore
from nucypher.keystore.db import Base
from nucypher.keystore.keypairs import SigningKeypair
from nucypher.utilities.sandbox.blockchain import TesterBlockchain, token_airdrop
from nucypher.utilities.sandbox.constants import (DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
DEVELOPMENT_TOKEN_AIRDROP_AMOUNT,
TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD)
from nucypher.utilities.sandbox.constants import (NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
DEVELOPMENT_TOKEN_AIRDROP_AMOUNT)
from nucypher.utilities.sandbox.middleware import MockRestMiddleware
from nucypher.utilities.sandbox.ursula import make_federated_ursulas, make_decentralized_ursulas
@ -59,7 +60,7 @@ def cleanup():
yield # we've got a lot of men and women here...
# Database teardown
for f in glob.glob("./**/*.db"): # TODO: Needs cleanup
for f in glob.glob("**/*.db"): # TODO: Needs cleanup
os.remove(f)
# Temp Storage Teardown
@ -90,8 +91,7 @@ def temp_config_root(temp_dir_path):
"""
User is responsible for closing the file given at the path.
"""
default_node_config = NodeConfiguration(temp=True,
auto_initialize=False,
default_node_config = NodeConfiguration(dev_mode=True,
config_root=temp_dir_path,
import_seed_registry=False)
yield default_node_config.config_root
@ -121,29 +121,23 @@ def certificates_tempdir():
@pytest.fixture(scope="module")
def ursula_federated_test_config():
ursula_config = UrsulaConfiguration(temp=True,
auto_initialize=True,
auto_generate_keys=True,
passphrase=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
ursula_config = UrsulaConfiguration(dev_mode=True,
is_me=True,
start_learning_now=False,
abort_on_learning_error=True,
federated_only=True,
network_middleware=MockRestMiddleware(),
save_metadata=False,
load_metadata=False)
reload_metadata=False)
yield ursula_config
ursula_config.cleanup()
@pytest.fixture(scope="module")
@pytest.mark.usefixtures('three_agents')
def ursula_decentralized_test_config(three_agents):
token_agent, miner_agent, policy_agent = three_agents
ursula_config = UrsulaConfiguration(temp=True,
auto_initialize=True,
auto_generate_keys=True,
passphrase=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
ursula_config = UrsulaConfiguration(dev_mode=True,
is_me=True,
start_learning_now=False,
abort_on_learning_error=True,
@ -151,24 +145,21 @@ def ursula_decentralized_test_config(three_agents):
network_middleware=MockRestMiddleware(),
import_seed_registry=False,
save_metadata=False,
load_metadata=False)
reload_metadata=False)
yield ursula_config
ursula_config.cleanup()
@pytest.fixture(scope="module")
def alice_federated_test_config(federated_ursulas):
config = AliceConfiguration(temp=True,
auto_initialize=True,
auto_generate_keys=True,
passphrase=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
config = AliceConfiguration(dev_mode=True,
is_me=True,
network_middleware=MockRestMiddleware(),
known_nodes=federated_ursulas,
federated_only=True,
abort_on_learning_error=True,
save_metadata=False,
load_metadata=False)
reload_metadata=False)
yield config
config.cleanup()
@ -178,34 +169,28 @@ def alice_blockchain_test_config(blockchain_ursulas, three_agents):
token_agent, miner_agent, policy_agent = three_agents
etherbase, alice_address, bob_address, *everyone_else = token_agent.blockchain.interface.w3.eth.accounts
config = AliceConfiguration(temp=True,
config = AliceConfiguration(dev_mode=True,
is_me=True,
auto_initialize=True,
auto_generate_keys=True,
checksum_address=alice_address,
passphrase=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
checksum_public_address=alice_address,
network_middleware=MockRestMiddleware(),
known_nodes=blockchain_ursulas,
abort_on_learning_error=True,
import_seed_registry=False,
save_metadata=False,
load_metadata=False)
reload_metadata=False)
yield config
config.cleanup()
@pytest.fixture(scope="module")
def bob_federated_test_config():
config = BobConfiguration(temp=True,
auto_initialize=True,
auto_generate_keys=True,
passphrase=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
config = BobConfiguration(dev_mode=True,
network_middleware=MockRestMiddleware(),
start_learning_now=False,
abort_on_learning_error=True,
federated_only=True,
save_metadata=False,
load_metadata=False)
reload_metadata=False)
yield config
config.cleanup()
@ -215,11 +200,8 @@ def bob_blockchain_test_config(blockchain_ursulas, three_agents):
token_agent, miner_agent, policy_agent = three_agents
etherbase, alice_address, bob_address, *everyone_else = token_agent.blockchain.interface.w3.eth.accounts
config = BobConfiguration(temp=True,
auto_initialize=True,
auto_generate_keys=True,
checksum_address=bob_address,
passphrase=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
config = BobConfiguration(dev_mode=True,
checksum_public_address=bob_address,
network_middleware=MockRestMiddleware(),
known_nodes=blockchain_ursulas,
start_learning_now=False,
@ -227,7 +209,7 @@ def bob_blockchain_test_config(blockchain_ursulas, three_agents):
federated_only=False,
import_seed_registry=False,
save_metadata=False,
load_metadata=False)
reload_metadata=False)
yield config
config.cleanup()
@ -241,7 +223,7 @@ def idle_federated_policy(federated_alice, federated_bob):
"""
Creates a Policy, in a manner typical of how Alice might do it, with a unique label
"""
n = DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
n = NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
random_label = b'label://' + os.urandom(32)
policy = federated_alice.create_policy(federated_bob, label=random_label, m=3, n=n, federated=True)
return policy
@ -250,7 +232,7 @@ def idle_federated_policy(federated_alice, federated_bob):
@pytest.fixture(scope="module")
def enacted_federated_policy(idle_federated_policy, federated_ursulas):
# Alice has a policy in mind and knows of enough qualifies Ursulas; she crafts an offer for them.
deposit = constants.NON_PAYMENT
deposit = NON_PAYMENT
contract_end_datetime = maya.now() + datetime.timedelta(days=5)
network_middleware = MockRestMiddleware()
@ -277,7 +259,7 @@ def idle_blockchain_policy(blockchain_alice, blockchain_bob):
@pytest.fixture(scope="module")
def enacted_blockchain_policy(idle_blockchain_policy, blockchain_ursulas):
# Alice has a policy in mind and knows of enough qualifies Ursulas; she crafts an offer for them.
deposit = constants.NON_PAYMENT(b"0000000")
deposit = NON_PAYMENT(b"0000000")
contract_end_datetime = maya.now() + datetime.timedelta(days=5)
network_middleware = MockRestMiddleware()
@ -331,7 +313,7 @@ def blockchain_bob(bob_blockchain_test_config):
@pytest.fixture(scope="module")
def federated_ursulas(ursula_federated_test_config):
_ursulas = make_federated_ursulas(ursula_config=ursula_federated_test_config,
quantity=DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK)
quantity=NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK)
yield _ursulas
@ -340,7 +322,7 @@ def blockchain_ursulas(three_agents, ursula_decentralized_test_config):
token_agent, miner_agent, policy_agent = three_agents
etherbase, alice, bob, *all_yall = token_agent.blockchain.interface.w3.eth.accounts
ursula_addresses = all_yall[:DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK]
ursula_addresses = all_yall[:NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK]
token_airdrop(origin=etherbase,
addresses=ursula_addresses,
@ -382,7 +364,7 @@ def testerchain(solidity_compiler):
# Create the blockchain
testerchain = TesterBlockchain(interface=deployer_interface,
test_accounts=DEFAULT_NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
test_accounts=NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK,
airdrop=False)
origin, *everyone = testerchain.interface.w3.eth.accounts

15
tests/network/test_network_actors.py
View File

@ -14,6 +14,8 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import os
import pytest
@ -26,7 +28,7 @@ from nucypher.characters.unlawful import Vladimir
from nucypher.crypto.api import keccak_digest
from nucypher.crypto.powers import SigningPower
from nucypher.network.nicknames import nickname_from_seed
from nucypher.utilities.sandbox.constants import TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
from nucypher.utilities.sandbox.constants import INSECURE_DEVELOPMENT_PASSWORD
from nucypher.utilities.sandbox.middleware import MockRestMiddleware
@ -41,8 +43,8 @@ def test_all_blockchain_ursulas_know_about_all_other_ursulas(blockchain_ursulas,
if address == propagating_ursula.checksum_public_address:
continue
else:
assert address in propagating_ursula.known_nodes, "{} did not know about {}".format(propagating_ursula,
nickname_from_seed(address))
assert address in propagating_ursula.known_nodes.addresses(), "{} did not know about {}".format(propagating_ursula,
nickname_from_seed(address))
@pytest.mark.slow()
@ -191,11 +193,16 @@ def test_alice_refuses_to_make_arrangement_unless_ursula_is_valid(blockchain_ali
message = vladimir._signable_interface_info_message()
signature = vladimir._crypto_power.power_ups(SigningPower).sign(message)
vladimir.substantiate_stamp(passphrase=TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD)
vladimir.substantiate_stamp(password=INSECURE_DEVELOPMENT_PASSWORD)
vladimir._interface_signature_object = signature
class FakeArrangement:
federated = False
ursula = target
vladimir.node_storage.store_node_certificate(host=target.rest_information()[0].host,
certificate=target.certificate,
checksum_address=target.checksum_public_address)
with pytest.raises(vladimir.InvalidNode):
idle_blockchain_policy.consider_arrangement(network_middleware=blockchain_alice.network_middleware,

8
tests/network/test_node_storage.py
View File

@ -26,8 +26,10 @@ from nucypher.utilities.sandbox.ursula import make_federated_ursulas
@pytest_twisted.inlineCallbacks
def test_one_node_stores_a_bunch_of_others(federated_ursulas, ursula_federated_test_config):
the_chosen_seednode = list(federated_ursulas)[2]
the_chosen_seednode = list(federated_ursulas)[2] # ...neo?
seed_node = the_chosen_seednode.seed_node_metadata()
newcomer = make_federated_ursulas(
ursula_config=ursula_federated_test_config,
quantity=1,
@ -41,9 +43,9 @@ def test_one_node_stores_a_bunch_of_others(federated_ursulas, ursula_federated_t
newcomer.start_learning_loop()
start = maya.now()
# Loop until the_chosen_seednode is in storage.
while not the_chosen_seednode in newcomer.node_storage.all(federated_only=True):
while the_chosen_seednode not in newcomer.node_storage.all(federated_only=True):
passed = maya.now() - start
if passed.seconds > 2:
if passed.seconds > 5:
pytest.fail("Didn't find the seed node.")
yield deferToThread(start_lonely_learning_loop)

25
tests/network/test_ursula_status.py Normal file
View File

@ -0,0 +1,25 @@
from nucypher.config.characters import UrsulaConfiguration
def test_render_lonely_ursula_status_page(tmpdir):
ursula_config = UrsulaConfiguration(dev_mode=True, federated_only=True)
ursula = ursula_config()
template = ursula.rest_server.routes._status_template
rendering = template.render(this_node=ursula, known_nodes=ursula.known_nodes)
assert '<!DOCTYPE html>' in rendering
assert ursula.nickname in rendering
def test_render_ursula_status_page_with_known_nodes(tmpdir, federated_ursulas):
ursula_config = UrsulaConfiguration(dev_mode=True, federated_only=True, known_nodes=federated_ursulas)
ursula = ursula_config()
template = ursula.rest_server.routes._status_template
rendering = template.render(this_node=ursula, known_nodes=ursula.known_nodes)
assert '<!DOCTYPE html>' in rendering
assert ursula.nickname in rendering
# Every known nodes address is rendered
for known_ursula in federated_ursulas:
assert known_ursula.checksum_public_address in rendering

25
tests/cli/test_main.py → tests/run_tests.py
View File

@ -16,14 +16,25 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from click.testing import CliRunner
import os
from nucypher.cli import cli
import maya
import pytest
def test_help_message():
runner = CliRunner()
result = runner.invoke(cli, ['--help'], catch_exceptions=False)
class NucypherPytestRunner:
TEST_PATH = os.path.join('tests', 'cli')
PYTEST_ARGS = ['--verbose', TEST_PATH]
assert result.exit_code == 0
assert 'Usage: cli [OPTIONS] COMMAND [ARGS]' in result.output, 'Missing or invalid help text was produced.'
def pytest_sessionstart(self):
print("*** Running Nucypher CLI Tests ***")
self.start_time = maya.now()
def pytest_sessionfinish(self):
duration = maya.now() - self.start_time
print("*** Nucypher Test Run Report ***")
print("""Run Duration ... {}""".format(duration))
def run():
pytest.main(NucypherPytestRunner.PYTEST_ARGS, plugins=[NucypherPytestRunner()])