Merge pull request #377 from KPrasch/config

Nucypher Configuration Parsers
pull/415/head
Tux 2018-08-29 01:38:49 -07:00 committed by GitHub
commit 2e44e0f3d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 320 additions and 292 deletions

38
.nucypher.ini Normal file
View File

@ -0,0 +1,38 @@
[nucypher]
[character]
start_learning_on_same_thread=True
always_be_learning=True
abort_on_learning_error=False
federated=False
ethereum_address=0xdeadbeef
[network.cryptography]
tls_curve=secp256r1
[ursula]
wallet_address=0xdeadbeef
stake=0
[ursula.network.rest]
host=127.0.0.1
port=5115
db_path=ursula.db.sqlite3
[ursula.network.dht]
host=127.0.0.1
port=5867
[blockchain]
tester=True
test_accounts=10
deploy=True
compile=True
temporary_registry=True
timeout=120
[blockchain.provider]
tester=False
type=ipc
ipc_path=/tmp/geth.ipc
poa=True

View File

@ -17,10 +17,16 @@ class EthereumContractAgent(ABC):
principal_contract_name = NotImplemented
__contract_address = NotImplemented
__instance = None
class ContractNotDeployed(Exception):
pass
def __new__(cls, *args, **kwargs):
if cls.__instance is None:
cls.__instance = super(EthereumContractAgent, cls).__new__(cls)
return cls.__instance
def __init__(self, blockchain: Blockchain=None, *args, **kwargs):
if blockchain is None:
@ -31,7 +37,6 @@ class EthereumContractAgent(ABC):
contract = self.blockchain.interface.get_contract_by_name(name=self.principal_contract_name,
upgradeable=self._upgradeable)
self.__contract = contract
super().__init__()
def __repr__(self):
@ -63,6 +68,7 @@ class EthereumContractAgent(ABC):
class NucypherTokenAgent(EthereumContractAgent):
principal_contract_name = "NuCypherToken"
_upgradeable = False
__instance = None
def approve_transfer(self, amount: int, target_address: str, sender_address: str) -> str:
"""Approve the transfer of token from the sender address to the target address."""
@ -83,6 +89,7 @@ class MinerAgent(EthereumContractAgent):
principal_contract_name = "MinersEscrow"
_upgradeable = True
__instance = None
class NotEnoughMiners(Exception):
pass
@ -218,6 +225,7 @@ class PolicyAgent(EthereumContractAgent):
principal_contract_name = "PolicyManager"
_upgradeable = True
__instance = None
def __init__(self, miner_agent: MinerAgent, *args, **kwargs):
super().__init__(blockchain=miner_agent.blockchain, *args, **kwargs)

View File

@ -81,24 +81,26 @@ class TesterBlockchain(Blockchain):
__default_num_test_accounts = 10
_default_network = 'tester'
def __init__(self, test_accounts=None, poa=False, airdrop=False, *args, **kwargs):
def __init__(self, test_accounts=None, poa=True, airdrop=False, *args, **kwargs):
# Depends on circumflex
super().__init__(*args, **kwargs)
# For use with Proof Of Authority test-blockchains
# For use with Proof-Of-Authority test-blockchains
if poa is True:
w3 = self.interface.w3
w3.middleware_stack.inject(geth_poa_middleware, layer=0)
# Generate additional ethereum accounts for testing
if len(self.interface.w3.eth.accounts) == 1:
enough_accounts = len(self.interface.w3.eth.accounts) > self.__default_num_test_accounts
if test_accounts is not None and not enough_accounts:
from tests.blockchain.eth import utilities
accounts_to_make = self.__default_num_test_accounts - len(self.interface.w3.eth.accounts)
test_accounts = test_accounts if test_accounts is not None else self.__default_num_test_accounts
utilities.generate_accounts(w3=self.interface.w3, quantity=test_accounts-1)
utilities.generate_accounts(w3=self.interface.w3, quantity=accounts_to_make)
assert test_accounts == len(self.interface.w3.eth.accounts)
assert test_accounts == len(self.interface.w3.eth.accounts)
if airdrop is True: # ETH for everyone!
one_million_ether = 10 ** 6 * 10 ** 18 # wei -> ether

View File

@ -23,7 +23,6 @@ from umbral.signing import Signature
from nucypher.blockchain.eth.actors import PolicyAuthor, Miner
from nucypher.blockchain.eth.agents import MinerAgent
from nucypher.config.configs import CharacterConfiguration
from nucypher.crypto.api import keccak_digest, encrypt_and_sign
from nucypher.crypto.constants import PUBLIC_ADDRESS_LENGTH, PUBLIC_KEY_LENGTH
from nucypher.crypto.kits import UmbralMessageKit
@ -68,7 +67,7 @@ class Character:
crypto_power: CryptoPower = None,
crypto_power_ups=None,
federated_only=False,
config: CharacterConfiguration = None,
config = None,
checksum_address: bytes = None,
always_be_learning=False,
start_learning_on_same_thread=False,

View File

@ -1,177 +0,0 @@
import json
import os
import warnings
from pathlib import Path
from typing import List
from web3 import IPCProvider
from nucypher.blockchain.eth.chains import Blockchain, TesterBlockchain
class NucypherConfiguration:
_default_configuration_directory = os.path.join(str(Path.home()), '.nucypher')
_identifier = NotImplemented # used as json config key
class NucypherConfigurationError(RuntimeError):
pass
def __init__(self, base_directory: str=None):
self.base_directory = base_directory or self._default_configuration_directory
def _save(self, path: str=None):
raise NotImplementedError
@classmethod
def _load(cls, path: str=None):
"""Instantiate a configuration object by reading from saved json file data"""
with open(path or cls._default_configuration_directory, 'r') as config:
data_dump = json.loads(config.read())
try:
subconfiguration_data = data_dump[cls._identifier]
except KeyError:
raise cls.NucypherConfigurationError('No saved configuration for {}'.format(cls._identifier))
try:
instance = cls(**subconfiguration_data)
except ValueError: # TODO: Correct exception?
raise cls.NucypherConfigurationError("Invalid configuration file data: {}.".format(subconfiguration_data))
return instance
class PolicyConfiguration(NucypherConfiguration):
"""Preferences regarding the authoring of new Policies, as Alice"""
_identifier = 'policy'
__default_m = 6 # TODO: Determine sensible values through experience
__default_n = 10
def __init__(self, default_m: int, default_n: int, *args, **kwargs):
self.prefered_m = default_m or self.__default_m
self.prefered_n = default_n or self.__default_n
super().__init__(*args, **kwargs)
class NetworkConfiguration(NucypherConfiguration):
"""Network configuration class for all things network transport"""
_identifier = 'network'
# Database
__default_db_name = 'nucypher_datastore.db' # TODO: choose database filename
__default_db_path = os.path.join(NucypherConfiguration._default_configuration_directory, __default_db_name)
# DHT Server
__default_dht_port = 5867
# REST Server
__default_ip_address = '127.0.0.1'
__default_rest_port = 5115 # TODO: choose a default rest port
def __init__(self, ip_address: str=None, rest_port: int=None,
dht_port: int=None, db_name: str=None, *args, **kwargs):
# Database
self.db_name = db_name or self.__default_db_name
# self.__db_path = db_path or self.__default_db_path # Sqlite
# DHT Server
self.dht_port = dht_port or self.__default_dht_port
# Rest Server
self.ip_address = ip_address or self.__default_ip_address
self.rest_port = rest_port or self.__default_rest_port
super().__init__(*args, **kwargs)
class BlockchainConfiguration(NucypherConfiguration):
"""
Blockchain configuration class, takes and preserves
the state of Web3 (and thus the blockchain) provider objects during runtime.
Network Name
==============
Network names are used primarily for the ethereum contract registry,
but also are sometimes used in determining the network configuration.
Geth networks
-------------
mainnet: Connect to the public ethereum mainnet via geth.
ropsten: Connect to the public ethereum ropsten testnet via geth.
temp: Local private chain whos data directory is removed when the chain is shutdown. Runs via geth.
Development Chains
------------------
tester: Ephemeral in-memory chain backed by pyethereum, pyevm, etc.
testrpc: Ephemeral in-memory chain for testing RPC calls
"""
_identifier = 'blockchain'
# Blockchain Network
__default_network = 'tester'
__default_timeout = 120 # seconds
__default_transaction_gas_limit = 500000 # TODO: determine sensible limit
def __init__(self, wallet_address: str=None, network: str=None, timeout: int=None,
transaction_gas_limit=None, compiler=None, registrar=None,
deploy=False, tester=False, *args, **kwargs):
self.__network = network if network is not None else self.__default_network
self.timeout = timeout if timeout is not None else self.__default_timeout
self.transaction_gas_limit = transaction_gas_limit or self.__default_transaction_gas_limit
self.__user_wallet_addresses = list()
if wallet_address is not None:
self.__user_wallet_addresses.append(wallet_address)
super().__init__(*args, **kwargs)
#
# Wallets
#
@property
def wallet_addresses(self) -> List[str]:
return self.__user_wallet_addresses
def add_wallet_address(self, ether_address: str) -> None:
"""TODO: Validate"""
if len(ether_address) != 42: # includes 0x prefix
raise ValueError("Invalid ethereum address: {}".format(ether_address))
self.__user_wallet_addresses.append(ether_address)
class CharacterConfiguration(NucypherConfiguration):
"""Encapsulates all sub-configurations, preserves the configurable state of a single character."""
_identifier = 'character'
__default_configuration_root = NucypherConfiguration._default_configuration_directory
__default_json_config_filepath = os.path.join(__default_configuration_root, 'conf.json')
def __init__(self,
keyring=None,
blockchain_config: BlockchainConfiguration=None,
network_config: NetworkConfiguration=None,
policy_config: PolicyConfiguration=None,
configuration_root: str=None,
json_config_filepath: str=None,
*args, **kwargs):
# Check for custom paths
self.__configuration_root = configuration_root or self.__default_configuration_root
self.__json_config_filepath = json_config_filepath or self.__default_json_config_filepath
if blockchain_config is None:
blockchain_config = BlockchainConfiguration()
# Sub-configurations # Who needs it...
self.keyring = keyring # Everyone
self.blockchain = blockchain_config # Everyone
self.policy = policy_config # Alice / Ursula
self.network = network_config or NetworkConfiguration() # Ursula
super().__init__(*args, **kwargs)

View File

@ -1,11 +0,0 @@
"""
Public facing client interface
"""
from nucypher.config.keys import NucypherKeyring
def _bootstrap_config():
"""Do not actually use this."""
passphrase = input("Enter passphrase >> ")
return NucypherKeyring.generate(passphrase=passphrase)

View File

@ -1,5 +1,6 @@
import nacl
import json
import os
import stat
from base64 import urlsafe_b64encode
from pathlib import Path
from typing import ClassVar
@ -15,7 +16,7 @@ from umbral.keys import UmbralPrivateKey
from web3.auto import w3
from nucypher.config import utils
from nucypher.config.configs import _DEFAULT_CONFIGURATION_DIR
from nucypher.config.configs import _DEFAULT_CONFIGURATION_DIR, NucypherConfiguration
from nucypher.config.utils import _parse_keyfile, _save_private_keyfile, validate_passphrase, _save_public_keyfile
from nucypher.crypto.powers import SigningPower, EncryptingPower, CryptoPower
@ -25,6 +26,92 @@ w3.eth.enable_unaudited_features()
_CONFIG_ROOT = os.path.join(str(Path.home()), '.nucypher')
def _parse_keyfile(keypath: str):
"""Parses a keyfile and returns key metadata as a dict."""
with open(keypath, 'r') as keyfile:
try:
key_metadata = json.loads(keyfile)
except json.JSONDecodeError:
raise NucypherConfiguration.NucypherConfigurationError("Invalid data in keyfile {}".format(keypath))
else:
return key_metadata
def _save_private_keyfile(keypath: str, key_data: dict) -> str:
"""
Creates a permissioned keyfile and save it to the local filesystem.
The file must be created in this call, and will fail if the path exists.
Returns the filepath string used to write the keyfile.
Note: getting and setting the umask is not thread-safe!
See linux open docs: http://man7.org/linux/man-pages/man2/open.2.html
---------------------------------------------------------------------
O_CREAT - If pathname does not exist, create it as a regular file.
O_EXCL - Ensure that this call creates the file: if this flag is
specified in conjunction with O_CREAT, and pathname already
exists, then open() fails with the error EEXIST.
---------------------------------------------------------------------
"""
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL # Write, Create, Non-Existing
mode = stat.S_IRUSR | stat.S_IWUSR # 0o600
try:
keyfile_descriptor = os.open(path=keypath, flags=flags, mode=mode)
finally:
os.umask(0) # Set the umask to 0 after opening
# Write and destroy file descriptor reference
with os.fdopen(keyfile_descriptor, 'wb') as keyfile:
keyfile.write(json.dumps(key_data))
output_path = keyfile.name
# TODO: output_path is an integer, who knows why?
del keyfile_descriptor
return output_path
def _save_public_keyfile(keypath: str, key_data: bytes) -> str:
"""
Creates a permissioned keyfile and save it to the local filesystem.
The file must be created in this call, and will fail if the path exists.
Returns the filepath string used to write the keyfile.
Note: getting and setting the umask is not thread-safe!
See Linux open docs: http://man7.org/linux/man-pages/man2/open.2.html
---------------------------------------------------------------------
O_CREAT - If pathname does not exist, create it as a regular file.
O_EXCL - Ensure that this call creates the file: if this flag is
specified in conjunction with O_CREAT, and pathname already
exists, then open() fails with the error EEXIST.
---------------------------------------------------------------------
"""
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL # Write, Create, Non-Existing
mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH # 0o644
try:
keyfile_descriptor = os.open(path=keypath, flags=flags, mode=mode)
finally:
os.umask(0) # Set the umask to 0 after opening
# Write and destroy the file descriptor reference
with os.fdopen(keyfile_descriptor, 'wb') as keyfile:
# key data should be urlsafe_base64
keyfile.write(key_data)
output_path = keyfile.name
# TODO: output_path is an integer, who knows why?
del keyfile_descriptor
return output_path
def _derive_key_material_from_passphrase(salt: bytes, passphrase: str) -> bytes:
"""
Uses Scrypt derivation to derive a key for encrypting key material.

View File

@ -1,95 +1,17 @@
import json
import configparser
import os
import stat
from typing import Tuple
from .configs import NucypherConfiguration
from web3 import IPCProvider
from nucypher.blockchain.eth.chains import Blockchain, TesterBlockchain
from nucypher.blockchain.eth.interfaces import EthereumContractRegistry, DeployerCircumflex, ControlCircumflex
from nucypher.blockchain.eth.sol.compile import SolidityCompiler
from nucypher.blockchain.eth.utilities import TemporaryEthereumContractRegistry
def _save_private_keyfile(keypath: str, key_data: dict) -> str:
"""
Creates a permissioned keyfile and save it to the local filesystem.
The file must be created in this call, and will fail if the path exists.
Returns the filepath string used to write the keyfile.
Note: getting and setting the umask is not thread-safe!
See linux open docs: http://man7.org/linux/man-pages/man2/open.2.html
---------------------------------------------------------------------
O_CREAT - If pathname does not exist, create it as a regular file.
O_EXCL - Ensure that this call creates the file: if this flag is
specified in conjunction with O_CREAT, and pathname already
exists, then open() fails with the error EEXIST.
---------------------------------------------------------------------
"""
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL # Write, Create, Non-Existing
mode = stat.S_IRUSR | stat.S_IWUSR # 0o600
try:
keyfile_descriptor = os.open(path=keypath, flags=flags, mode=mode)
finally:
os.umask(0) # Set the umask to 0 after opening
# Write and destroy file descriptor reference
with os.fdopen(keyfile_descriptor, 'wb') as keyfile:
keyfile.write(json.dumps(key_data))
output_path = keyfile.name
# TODO: output_path is an integer, who knows why?
del keyfile_descriptor
return output_path
def _save_public_keyfile(keypath: str, key_data: bytes) -> str:
"""
Creates a permissioned keyfile and save it to the local filesystem.
The file must be created in this call, and will fail if the path exists.
Returns the filepath string used to write the keyfile.
Note: getting and setting the umask is not thread-safe!
See Linux open docs: http://man7.org/linux/man-pages/man2/open.2.html
---------------------------------------------------------------------
O_CREAT - If pathname does not exist, create it as a regular file.
O_EXCL - Ensure that this call creates the file: if this flag is
specified in conjunction with O_CREAT, and pathname already
exists, then open() fails with the error EEXIST.
---------------------------------------------------------------------
"""
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL # Write, Create, Non-Existing
mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH # 0o644
try:
keyfile_descriptor = os.open(path=keypath, flags=flags, mode=mode)
finally:
os.umask(0) # Set the umask to 0 after opening
# Write and destroy the file descriptor reference
with os.fdopen(keyfile_descriptor, 'wb') as keyfile:
# key data should be urlsafe_base64
keyfile.write(key_data)
output_path = keyfile.name
# TODO: output_path is an integer, who knows why?
del keyfile_descriptor
return output_path
def _parse_keyfile(keypath: str):
"""Parses a keyfile and returns key metadata as a dict."""
with open(keypath, 'r') as keyfile:
try:
key_metadata = json.loads(keyfile)
except json.JSONDecodeError:
raise NucypherConfiguration.NucypherConfigurationError("Invalid data in keyfile {}".format(keypath))
else:
return key_metadata
DEFAULT_CONFIG_DIR = "~"
DEFAULT_INI_FILEPATH = './.nucypher.ini'
def generate_confg_dir(path: str=None,) -> None:
@ -97,8 +19,7 @@ def generate_confg_dir(path: str=None,) -> None:
Create the configuration directory tree.
If the directory already exists, FileExistsError is raised.
"""
path = path if path else NucypherConfiguration._default_configuration_directory
path = path if path else DEFAULT_CONFIG_DIR
if not os.path.exists(path):
os.mkdir(path, mode=0o755)
@ -112,12 +33,12 @@ def validate_passphrase(passphrase) -> bool:
for rule, failure_message in rules:
if not rule:
raise NucypherConfiguration.NucypherConfigurationError(failure_message)
raise RuntimeError(failure_message)
return True
def check_config_tree(configuration_dir: str=None) -> bool:
path = configuration_dir if configuration_dir else NucypherConfiguration._default_configuration_directory
path = configuration_dir if configuration_dir else DEFAULT_CONFIG_DIR
if not os.path.exists(path):
raise FileNotFoundError('No NuCypher configuration directory found at {}.'.format(configuration_dir))
return True
@ -134,3 +55,165 @@ def check_config_runtime() -> bool:
return True
def validate_nucypher_ini_config(config=None,
filepath: str=DEFAULT_INI_FILEPATH,
raise_on_failure: bool=False) -> Tuple[bool, list]:
if config is None:
config = configparser.ConfigParser()
try:
config.read(filepath)
except:
raise # FIXME
required_sections = ("provider", "nucypher")
missing_sections = list()
for section in required_sections:
if section not in config.sections():
missing_sections.append(section)
if raise_on_failure is True:
raise RuntimeError("Invalid config file")
else:
if len(missing_sections) > 0:
return False, missing_sections
def parse_blockchain_config(config=None, filepath: str=DEFAULT_INI_FILEPATH) -> dict:
if config is None:
config = configparser.ConfigParser()
config.read(filepath)
providers = list()
if config['provider']['type'] == 'ipc':
try:
provider = IPCProvider(config['provider']['ipc_path'])
except KeyError:
message = "ipc_path must be provided when using an IPC provider"
raise Exception(message) # FIXME
else:
providers.append(provider)
else:
raise NotImplementedError
poa = config.getboolean(section='provider', option='poa', fallback=True)
tester = config.getboolean(section='blockchain', option='tester', fallback=False)
test_accounts = config.getint(section='blockchain', option='test_accounts', fallback=0)
deploy = config.getboolean(section='blockchain', option='deploy', fallback=False)
compile = config.getboolean(section='blockchain', option='compile', fallback=False)
timeout = config.getint(section='blockchain', option='timeout', fallback=10)
tmp_registry = config.getboolean(section='blockchain', option='temporary_registry', fallback=False)
registry_filepath = config.get(section='blockchain', option='registry_filepath', fallback='.registry.json')
#
# Initialize
#
compiler = SolidityCompiler() if compile else None
if tmp_registry:
registry = TemporaryEthereumContractRegistry()
else:
registry = EthereumContractRegistry(registry_filepath=registry_filepath)
interface_class = ControlCircumflex if not deploy else DeployerCircumflex
circumflex = interface_class(timeout=timeout,
providers=providers,
compiler=compiler,
registry=registry)
if tester:
blockchain = TesterBlockchain(interface=circumflex,
poa=poa,
test_accounts=test_accounts,
airdrop=True)
else:
blockchain = Blockchain(interface=circumflex)
blockchain_payload = dict(compiler=compiler,
registry=registry,
interface=circumflex,
blockchain=blockchain,
tester=tester,
test_accounts=test_accounts,
deploy=deploy,
poa=poa,
timeout=timeout,
tmp_registry=tmp_registry,
registry_filepath=registry_filepath)
return blockchain_payload
def parse_character_config(config=None, filepath: str=DEFAULT_INI_FILEPATH):
if config is None:
config = configparser.ConfigParser()
config.read(filepath)
character_payload = dict(start_learning_on_same_thread=config.getboolean(section='nucypher.character', option='temporary_registry'),
abort_on_learning_error=config.getboolean(section='nucypher.character', option='abort_on_learning_error'),
federated_only=config.getboolean(section='nucypher.character', option='federated'),
checksum_address=config.get(section='nucypher.character', option='ethereum_address'),
always_be_learning=config.getboolean(section='nucypher.character', option='always_be_learning'))
return character_payload
def parse_ursula_config(config=None, filepath: str=DEFAULT_INI_FILEPATH):
if config is None:
config = configparser.ConfigParser()
config.read(filepath)
if "stake" in config.sections():
try:
stake_index = int(config["ursula"]["stake"])
except ValueError:
stakes = []
stake_index_tags = {'latest': len(stakes),
'only': stakes[0]}
raise NotImplementedError
ursula_payload = dict(checksum_address=config.get(section='ursula', option='wallet_address'),
# Rest
rest_host=config.get(section='ursula.network.rest', option='rest_host'),
rest_port=config.getint(section='ursula.network.rest', option='rest_port'),
db_name=config.get(section='ursula.network.rest', option='db_name'),
# DHT
dht_host=config.get(section='ursula.network.dht', option='dht_host'),
dht_port=config.getint(section='ursula.network.dht', option='dht_port'))
return ursula_payload
def parse_nucypher_ini_config(filepath: str=DEFAULT_INI_FILEPATH) -> dict:
"""Top-level parser with sub-parser routing"""
validate_nucypher_ini_config(filepath=filepath, raise_on_failure=True)
config = configparser.ConfigParser()
config.read(filepath)
# Parser router
parsers = {"character": parse_character_config,
"blockchain": parse_blockchain_config,
"ursula": parse_ursula_config,
}
staged_payloads = set()
for section, parser in parsers.items():
section_payload = parser(config)
staged_payloads.add(section_payload)
payload = dict()
for payload in staged_payloads:
payload.update(payload)
return payload

View File

@ -13,7 +13,6 @@ from kademlia.utils import digest
from bytestring_splitter import VariableLengthBytestring
from constant_sorrow import constants
from hendrix.experience import crosstown_traffic
from nucypher.config.configs import NetworkConfiguration
from nucypher.crypto.kits import UmbralMessageKit
from nucypher.crypto.powers import SigningPower, TLSHostingPower
from nucypher.keystore.keypairs import HostingKeypair
@ -120,7 +119,7 @@ class ProxyRESTServer:
self._crypto_power.consume_power_up(tls_hosting_power)
@classmethod
def from_config(cls, network_config: NetworkConfiguration = None):
def from_config(cls, network_config = None):
"""Create a server object from config values, or from a config file."""
# if network_config is None:
# NetworkConfiguration._load()