Merge pull request #476 from KPrasch/storages

Configurable Node Storages

.circleci/config.yml

@@ -5,37 +5,45 @@ workflows:
   test:
     jobs:
       - bundle_dependencies-36
-      - mypy_type_check:
+      - eth_contract_unit:
           requires:
             - bundle_dependencies-36
-      - eth_contract_unit:
+      - config_unit:
           requires:
             - bundle_dependencies-36
       - crypto_unit:
           requires:
-              - bundle_dependencies-36
+            - bundle_dependencies-36
       - network_unit:
           requires:
-              - bundle_dependencies-36
+            - bundle_dependencies-36
       - keystore_unit:
           requires:
-              - bundle_dependencies-36
+            - bundle_dependencies-36
       - blockchain_interface_unit:
           requires:
-              - bundle_dependencies-36
+            - bundle_dependencies-36
       - character:
           requires:
-              - crypto_unit
-              - network_unit
-              - keystore_unit
+            - bundle_dependencies-36
       - intercontract_integration:
           requires:
             - eth_contract_unit
-      - nucypher_cli_tests:
+      - mypy_type_check:
           requires:
-              - crypto_unit
-              - network_unit
-              - keystore_unit
+            - config_unit
+            - crypto_unit
+            - network_unit
+            - keystore_unit
+            - character
+      - cli_tests:
+          requires:
+            - blockchain_interface_unit
+            - config_unit
+            - crypto_unit
+            - network_unit
+            - keystore_unit
+            - character
 
 python_36_base: &python_36_base
     working_directory: ~/nucypher-depends
@@ -50,7 +58,7 @@ jobs:
       - run:
           name: Install Python Dependencies with Pipenv
           command: |
-            pip3 install pip==18.0
+            pip3 install --user pip==18.0
             pipenv install --three --dev --skip-lock --verbose
       - run:
           name: Install Solidity Compiler
@@ -82,7 +90,19 @@ jobs:
       - run:
           name: Ethereum Contract Unit Tests
           command: |
-            pipenv run pytest --junitxml=./reports/pytest/eth-contract-unit-report.xml -v --runslow $(circleci tests glob tests/blockchain/eth/contracts/*/test_*.py | circleci tests split --split-by=timings)
+            pipenv run pytest --junitxml=./reports/pytest/eth-contract-unit-report.xml -v --runslow $(circleci tests glob tests/blockchain/eth/contracts/**/**/test_*.py | circleci tests split --split-by=timings)
+      - store_test_results:
+          path: ./reports/pytest/
+
+  config_unit:
+    <<: *python_36_base
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ~/.local/share/virtualenvs/
+      - run:
+          name: Node Configuration Tests
+          command: pipenv run pytest --cov=nucypher/config -v --runslow tests/config --junitxml=./reports/pytest/results.xml
       - store_test_results:
           path: ./reports/pytest/
 
@@ -147,7 +167,7 @@ jobs:
       - store_test_results:
           path: ./reports/pytest/
 
-  nucypher_cli_tests:
+  cli_tests:
     <<: *python_36_base
     steps:
     - checkout
@@ -171,7 +191,7 @@ jobs:
           command: |
               pipenv run pip install lxml
       - run:
-          name: Run Mypy Static Type Checks
+          name: Run Mypy Static Type Checks (Always Succeed)
           command: |
               mkdir ./mypy_reports ./mypy_results
               export MYPYPATH=./nucypher

Pipfile

@@ -51,6 +51,11 @@ py-solc = "*"
 #eth-tester = "*"
 eth-tester = {git = "https://github.com/KPrasch/eth-tester.git", ref = "ef4bb2fa793af8aa964b83536b20f525aa74d4e4"}
 py-evm = ">=0.2.0a31"
+#
+# CLI
+#
+moto = "*"
+boto3 = "*"
 nucypher = {editable = true, path = "."}
 
 [pipenv]

cli/main.py

@@ -29,25 +29,14 @@ from nucypher.config.characters import UrsulaConfiguration
 from nucypher.config.constants import BASE_DIR
 from nucypher.config.keyring import NucypherKeyring
 from nucypher.config.node import NodeConfiguration
-from nucypher.config.utils import validate_configuration_file, generate_local_wallet, generate_account
 from nucypher.utilities.sandbox.blockchain import TesterBlockchain, token_airdrop
 from nucypher.utilities.sandbox.constants import (DEVELOPMENT_TOKEN_AIRDROP_AMOUNT,
                                                   DEVELOPMENT_ETH_AIRDROP_AMOUNT,
                                                   DEFAULT_SIMULATION_REGISTRY_FILEPATH)
 from nucypher.utilities.sandbox.ursula import UrsulaProcessProtocol
 
+
 __version__ = '0.1.0-alpha.0'
-
-
-def echo_version(ctx, param, value):
-    if not value or ctx.resilient_parsing:
-        return
-    click.secho(__version__, bold=True)
-    ctx.exit()
-
-
-DEBUG = True
-
 BANNER = """
                                   _               
                                  | |              
@@ -63,10 +52,14 @@ BANNER = """
 """.format(__version__)
 
 
-#
-# Setup Logging
-#
+def echo_version(ctx, param, value):
+    if not value or ctx.resilient_parsing:
+        return
+    click.secho(BANNER, bold=True)
+    ctx.exit()
 
+
+# Setup Logging
 root = logging.getLogger()
 root.setLevel(logging.DEBUG)
 
@@ -81,6 +74,12 @@ root.addHandler(ch)
 # CLI Configuration
 #
 
+# CLI Constants
+DEBUG = True
+KEYRING_PASSPHRASE_ENVVAR = 'NUCYPHER_KEYRING_PASSPHRASE'
+
+
+# Pending Configuration Named Tuple
 fields = 'passphrase wallet signing tls skip_keys save_file'.split()
 PendingConfigurationDetails = collections.namedtuple('PendingConfigurationDetails', fields)
 
@@ -120,7 +119,7 @@ class NucypherClickConfig:
                                                      config_root=self.config_root,
                                                      auto_initialize=False)
         elif self.dev:
-            node_configuration = configuration_class(temp=self.dev, auto_initialize=False)
+            node_configuration = configuration_class(temp=self.dev, auto_initialize=False, federated_only=self.federated_only)
         elif self.config_file:
             click.echo("Using configuration file {}".format(self.config_file))
             node_configuration = configuration_class.from_configuration_file(filepath=self.config_file)
@@ -166,12 +165,15 @@ class NucypherClickConfig:
             passphrase = click.prompt(message, hide_input=True, confirmation_prompt=True)
 
         if choice == 'local':
-            keyring = generate_local_wallet(passphrase=passphrase, keyring_root=self.node_configuration.keyring_dir)
+            keyring = NucypherKeyring.generate(passphrase=passphrase,
+                                               keyring_root=self.node_configuration.keyring_dir,
+                                               encrypting=False,
+                                               wallet=True)
             new_address = keyring.checksum_address
         elif choice == 'hosted':
-            new_address = generate_account(w3=self.blockchain.interface.w3, passphrase=passphrase)
+            new_address = self.blockchain.interface.w3.personal.newAccount(passphrase)
         else:
-            raise click.Abort()
+            raise click.BadParameter("Invalid choice; Options are hosted or local.")
         return new_address
 
     def _collect_pending_configuration_details(self, ursula: bool=False, force: bool = False) -> PendingConfigurationDetails:
@@ -201,8 +203,8 @@ class NucypherClickConfig:
             if not any((generate_wallet, generate_tls_keys, generate_encrypting_keys)):
                 skip_all_key_generation = click.confirm("Skip all key generation (Provide custom configuration file)?")
         if not skip_all_key_generation:
-            if os.environ.get("NUCYPHER_KEYRING_PASSPHRASE"):
-                passphrase = os.environ.get("NUCYPHER_KEYRING_PASSPHRASE")
+            if os.environ.get(KEYRING_PASSPHRASE_ENVVAR):
+                passphrase = os.environ.get(KEYRING_PASSPHRASE_ENVVAR)
             else:
                 passphrase = click.prompt("Enter a passphrase to encrypt your keyring",
                                           hide_input=True, confirmation_prompt=True)
@@ -227,14 +229,19 @@ class NucypherClickConfig:
                 click.echo("Seed contract registry does not exist at path {}.  "
                            "Use --no-registry to skip.".format(registry_source))
                 raise click.Abort()
+
+        if self.config_root:  # Custom installation location
+            self.node_configuration.config_root = self.config_root
+        self.node_configuration.federated_only = self.federated_only
+
         try:
             pending_config = self._collect_pending_configuration_details(force=force, ursula=ursula)
-            new_installation_path = self.node_configuration.write(passphrase=pending_config.passphrase,
-                                                                  wallet=pending_config.wallet,
-                                                                  encrypting=pending_config.signing,
-                                                                  tls=pending_config.tls,
-                                                                  no_registry=no_registry,
-                                                                  no_keys=pending_config.skip_keys)
+            new_installation_path = self.node_configuration.initialize(passphrase=pending_config.passphrase,
+                                                                       wallet=pending_config.wallet,
+                                                                       encrypting=pending_config.signing,
+                                                                       tls=pending_config.tls,
+                                                                       no_registry=no_registry,
+                                                                       no_keys=pending_config.skip_keys)
             if not pending_config.skip_keys:
                 click.secho("Generated new keys at {}".format(self.node_configuration.keyring_dir), fg='blue')
         except NodeConfiguration.ConfigurationError as e:
@@ -370,9 +377,8 @@ def configure(config,
         is_valid = True      # Until there is a reason to believe otherwise...
         try:
             if filesystem:   # Check runtime directory
-                is_valid = NodeConfiguration.validate(config_root=config.node_configuration.config_root, no_registry=no_registry)
-            if config.config_file:
-                is_valid = validate_configuration_file(filepath=config.node_configuration.config_file_location)
+                is_valid = NodeConfiguration.validate(config_root=config.node_configuration.config_root,
+                                                      no_registry=no_registry)
         except NodeConfiguration.InvalidConfiguration:
             is_valid = False
         finally:
@@ -572,27 +578,28 @@ def stake(config,
                    start_period=start_period,
                    end_period=end_period))
 
-        if not click.confirm("Is this correct?"):
-            # field = click.prompt("Which stake field do you want to edit?")
-            raise NotImplementedError
-
-        # Initialize the staged stake
-        config.miner_agent.deposit_tokens(amount=value, lock_periods=duration, sender_address=address)
-
-        proc_params = ['run_ursula']
-        processProtocol = UrsulaProcessProtocol(command=proc_params)
-        ursula_proc = reactor.spawnProcess(processProtocol, "nucypher-cli", proc_params)
+        # TODO: Ursula Process management
+        # if not click.confirm("Is this correct?"):
+        #     # field = click.prompt("Which stake field do you want to edit?")
+        #     raise NotImplementedError
+        #
+        # # Initialize the staged stake
+        # config.miner_agent.deposit_tokens(amount=value, lock_periods=duration, sender_address=address)
+        #
+        # proc_params = ['run_ursula']
+        # processProtocol = UrsulaProcessProtocol(command=proc_params, checksum_address=checksum_address)
+        # ursula_proc = reactor.spawnProcess(processProtocol, "nucypher-cli", proc_params)
+        raise NotImplementedError
 
     elif action == 'resume':
         """Reconnect and resume an existing live stake"""
-
-        proc_params = ['run_ursula']
-        processProtocol = UrsulaProcessProtocol(command=proc_params)
-        ursula_proc = reactor.spawnProcess(processProtocol, "nucypher-cli", proc_params)
+        # proc_params = ['run_ursula']
+        # processProtocol = UrsulaProcessProtocol(command=proc_params, checksum_address=checksum_address)
+        # ursula_proc = reactor.spawnProcess(processProtocol, "nucypher-cli", proc_params)
+        raise NotImplementedError
 
     elif action == 'confirm-activity':
         """Manually confirm activity for the active period"""
-
         stakes = config.miner_agent.get_all_stakes(miner_address=address)
         if len(stakes) == 0:
             raise RuntimeError("There are no active stakes for {}".format(address))
@@ -645,7 +652,7 @@ def stake(config,
 
 
 @cli.command()
-@click.option('--geth', help="Simulate with geth", is_flag=True)
+@click.option('--geth', help="Simulate with geth dev-mode", is_flag=True)
 @click.option('--pyevm', help="Simulate with PyEVM", is_flag=True)
 @click.option('--nodes', help="The number of nodes to simulate", type=click.INT, default=10)
 @click.argument('action')
@@ -936,12 +943,10 @@ def deploy(config,
                 __deployer_init_args.update({dependant: __deployment_agents[dependant]})
 
             if upgradeable:
-                def __collect_secret_hash():
-                    secret = click.prompt("Enter secret hash for {}".format(__contract_name), hide_input=True, confirmation_prompt=True)
-                    secret_hash = hashlib.sha256(secret)
-                    __deployer_init_args.update({'secret_hash': secret_hash})
-                    return secret
-                __collect_secret_hash()
+                secret = click.prompt("Enter deployment secret for {}".format(__contract_name),
+                                      hide_input=True, confirmation_prompt=True)
+                secret_hash = hashlib.sha256(secret)
+                __deployer_init_args.update({'secret_hash': secret_hash})
 
             __deployer = deployer_class(**__deployer_init_args)
 
@@ -1006,7 +1011,7 @@ def deploy(config,
 
         if not force and click.confirm("Save transaction hashes to JSON file?"):
             file = click.prompt("Enter output filepath", type=click.File(mode='w'))   # TODO
-            file.write(json.dumps(__deployment_transactions))
+            file.__write(json.dumps(__deployment_transactions))
             click.secho("Successfully wrote transaction hashes file to {}".format(file.path), fg='green')
 
     else:
@@ -1163,12 +1168,12 @@ def ursula(config,
     config.operating_mode = "federated" if ursula_config.federated_only else "decentralized"
     click.secho("Running in {} mode".format(config.operating_mode), fg='blue')
 
-    if additional_nodes:  # Secondary override
-        ursula_config.read_known_nodes(known_metadata_dir=additional_nodes)
-        click.secho("Loaded additional known nodes", color='blue')
+    # ursula_config.read_known_nodes(known_metadata_dir=additional_nodes)
+    # if additional_nodes:  # Secondary override
+    #     click.secho("Loaded additional known nodes", color='blue')
 
     quantity_known_nodes = len(ursula_config.known_nodes)
-    if quantity_known_nodes:
+    if quantity_known_nodes > 0:
         click.secho("Loaded {} known nodes from storages".format(quantity_known_nodes, fg='blue'))
     else:
         click.secho("WARNING: No seed nodes available", fg='red', bold=True)

nucypher/characters/base.py

@@ -1,26 +1,27 @@
-import os
 import random
-import time
 from collections import defaultdict
 from collections import deque
 from contextlib import suppress
 from logging import Logger
 from logging import getLogger
-from typing import Dict, ClassVar, Set
-from typing import Tuple
-from typing import Union, List
+from tempfile import TemporaryDirectory
 
 import maya
 import requests
+import time
 from constant_sorrow import constants, default_constant_splitter
 from eth_keys import KeyAPI as EthKeyAPI
 from eth_utils import to_checksum_address, to_canonical_address
 from requests.exceptions import SSLError
 from twisted.internet import reactor
 from twisted.internet import task
+from typing import Dict, ClassVar, Set
+from typing import Tuple
+from typing import Union, List
 from umbral.keys import UmbralPublicKey
 from umbral.signing import Signature
 
+from nucypher.config.storages import InMemoryNodeStorage
 from nucypher.crypto.api import encrypt_and_sign
 from nucypher.crypto.kits import UmbralMessageKit
 from nucypher.crypto.powers import CryptoPower, SigningPower, EncryptingPower, NoSigningPower, CryptoPowerUp
@@ -40,8 +41,13 @@ class Learner:
 
     _SHORT_LEARNING_DELAY = 5
     _LONG_LEARNING_DELAY = 90
+    LEARNING_TIMEOUT = 10
     _ROUNDS_WITHOUT_NODES_AFTER_WHICH_TO_SLOW_DOWN = 10
 
+    # For Keeps
+    __DEFAULT_NODE_STORAGE = InMemoryNodeStorage
+    __DEFAULT_MIDDLEWARE_CLASS = RestMiddleware
+
     class NotEnoughTeachers(RuntimeError):
         pass
 
@@ -50,14 +56,15 @@ class Learner:
 
     def __init__(self,
                  common_name: str,
-                 network_middleware: RestMiddleware = RestMiddleware(),
+                 network_middleware: RestMiddleware = __DEFAULT_MIDDLEWARE_CLASS(),
                  start_learning_now: bool = False,
                  learn_on_same_thread: bool = False,
                  known_nodes: tuple = None,
                  known_certificates_dir: str = None,
-                 known_metadata_dir: str = None,
+                 node_storage = None,
                  save_metadata: bool = False,
-                 abort_on_learning_error: bool = False) -> None:
+                 abort_on_learning_error: bool = False
+                 ) -> None:
 
         self.log = getLogger("characters")                       # type: Logger
 
@@ -71,21 +78,25 @@ class Learner:
         self._learning_listeners = defaultdict(list)
         self._node_ids_to_learn_about_immediately = set()
 
-        self.known_certificates_dir = known_certificates_dir
+        self.known_certificates_dir = known_certificates_dir or TemporaryDirectory("nucypher-tmp-certs-").name
         self.__known_nodes = dict()
 
         # Read
-        self.known_metadata_dir = known_metadata_dir
-        if save_metadata and known_metadata_dir is None:
-            raise ValueError("Cannot save nodes without a known_metadata_dir")
+        if node_storage is None:
+            node_storage = self.__DEFAULT_NODE_STORAGE(federated_only=self.federated_only,    #TODO: remove federated_only
+                                                       character_class=self.__class__)
+
+        self.node_storage = node_storage
+        if save_metadata and node_storage is constants.NO_STORAGE_AVAILIBLE:
+            raise ValueError("Cannot save nodes without a configured node storage")
 
         known_nodes = known_nodes or tuple()
-        self.unresponsive_nodes = list()  # TODO: Attempt to use these again later
+        self.unresponsive_startup_nodes = list()  # TODO: Attempt to use these again later
         for node in known_nodes:
             try:
                 self.remember_node(node)
             except self.UnresponsiveTeacher:
-                self.unresponsive_nodes.append(node)
+                self.unresponsive_startup_nodes.append(node)
 
         self.teacher_nodes = deque()
         self._current_teacher_node = None   # type: Teacher
@@ -249,7 +260,7 @@ class Learner:
 
     def block_until_specific_nodes_are_known(self,
                                              canonical_addresses: Set,
-                                             timeout=10,
+                                             timeout=LEARNING_TIMEOUT,
                                              allow_missing=0,
                                              learn_on_this_thread=False):
         start = maya.now()
@@ -328,17 +339,7 @@ class Learner:
         # Scenario 3: We don't know about this node, and neither does our friend.
 
     def write_node_metadata(self, node, serializer=bytes) -> str:
-
-        try:
-            filename = "{}.node".format(node.checksum_public_address)
-        except AttributeError:
-            raise AttributeError("{} does not have a rest_interface attached".format(self))
-
-        metadata_filepath = os.path.join(self.known_metadata_dir, filename)
-        with open(metadata_filepath, "w") as f:
-            f.write(serializer(node).hex())
-        self.log.info("Wrote new node metadata {}".format(metadata_filepath))
-        return metadata_filepath
+        return self.node_storage.save(node=node)
 
     def learn_from_teacher_node(self, eager=True):
         """

nucypher/characters/lawful.py

@@ -2,7 +2,7 @@ import binascii
 import random
 from collections import OrderedDict
 from functools import partial
-from typing import Iterable
+from typing import Iterable, Callable
 from typing import List
 
 import maya
@@ -20,6 +20,7 @@ from umbral.signing import Signature
 from nucypher.blockchain.eth.actors import PolicyAuthor, Miner
 from nucypher.blockchain.eth.agents import MinerAgent
 from nucypher.characters.base import Character, Learner
+from nucypher.config.storages import NodeStorage
 from nucypher.crypto.api import keccak_digest
 from nucypher.crypto.constants import PUBLIC_ADDRESS_LENGTH, PUBLIC_KEY_LENGTH
 from nucypher.crypto.powers import SigningPower, EncryptingPower, DelegatingPower, BlockchainPower
@@ -397,7 +398,7 @@ class Ursula(Character, VerifiableNode, Miner):
                  rest_port: int,
                  certificate: Certificate = None,
                  certificate_filepath: str = None,
-                 tls_private_key = None,           # TODO: Derivie from keyring
+                 tls_private_key = None,           # TODO: Derive from keyring
 
                  db_name: str = None,
                  db_filepath: str = None,
@@ -682,12 +683,11 @@ class Ursula(Character, VerifiableNode, Miner):
         return stranger_ursulas
 
     @classmethod
-    def from_metadata_file(cls, filepath: str, federated_only: bool, *args, **kwargs) -> 'Ursula':
-        with open(filepath, "r") as seed_file:
-            seed_file.seek(0)
-            node_bytes = binascii.unhexlify(seed_file.read())
-            node = Ursula.from_bytes(node_bytes, federated_only=federated_only, *args, **kwargs)
-            return node
+    def from_storage(cls,
+                     node_storage: NodeStorage,
+                     checksum_adress: str,
+                     federated_only: bool = False, *args, **kwargs) -> 'Ursula':
+        return node_storage.get(checksum_address=checksum_adress)
 
     #
     # Properties

nucypher/config/blockchain.py

@@ -1,11 +0,0 @@
-from nucypher.config.parsers import parse_blockchain_config
-
-
-class BlockchainConfiguration:
-
-    def __init__(self):
-        pass
-
-    @classmethod
-    def from_config_file(cls, filepath: str):
-        parse_blockchain_config(filepath=filepath)

nucypher/config/characters.py

@@ -19,9 +19,9 @@ class UrsulaConfiguration(NodeConfiguration):
     _Character = Ursula
     _name = 'ursula'
     DEFAULT_CONFIG_FILE_LOCATION = os.path.join(DEFAULT_CONFIG_ROOT, '{}.config'.format(_name))
-
     DEFAULT_REST_HOST = '127.0.0.1'
     DEFAULT_REST_PORT = 9151
+
     __DB_TEMPLATE = "ursula.{port}.db"
     DEFAULT_DB_NAME = __DB_TEMPLATE.format(port=DEFAULT_REST_PORT)
 
@@ -85,11 +85,11 @@ class UrsulaConfiguration(NodeConfiguration):
         base_filepaths.update(filepaths)
         return base_filepaths
 
-    def write(self, tls: bool = True, *args, **kwargs):
-        return super().write(tls=tls,
-                             host=self.rest_host,
-                             curve=self.tls_curve,
-                             *args, **kwargs)
+    def initialize(self, tls: bool = True, *args, **kwargs):
+        return super().initialize(tls=tls,
+                                  host=self.rest_host,
+                                  curve=self.tls_curve,
+                                  *args, **kwargs)
 
     @property
     def static_payload(self) -> dict:

nucypher/config/constants.py

@@ -5,20 +5,13 @@ from appdirs import AppDirs
 
 import nucypher
 
-#
 # Base Filepaths
-#
+
 BASE_DIR = abspath(dirname(dirname(nucypher.__file__)))
 PROJECT_ROOT = abspath(dirname(nucypher.__file__))
 APP_DIR = AppDirs("nucypher", "NuCypher")
-
-#
-# Configuration File
-#
 DEFAULT_CONFIG_ROOT = APP_DIR.user_data_dir
 
-#
 # Test Constants  # TODO: Tidy up filepath here
-#
 TEST_CONTRACTS_DIR = os.path.join(BASE_DIR, 'tests', 'blockchain', 'eth', 'contracts', 'contracts')
 NUMBER_OF_URSULAS_IN_MOCK_NETWORK = 10

nucypher/config/node.py

@@ -1,33 +1,35 @@
+import binascii
 import json
 import os
-from glob import glob
 from json import JSONDecodeError
 from logging import getLogger
-from os.path import abspath
 from tempfile import TemporaryDirectory
-from typing import Set, List
+from typing import List
 
 from constant_sorrow import constants
 
-from nucypher.characters.base import Character
 from nucypher.config.constants import DEFAULT_CONFIG_ROOT, BASE_DIR
 from nucypher.config.keyring import NucypherKeyring
+from nucypher.config.storages import NodeStorage, InMemoryNodeStorage
 from nucypher.crypto.powers import CryptoPowerUp
 from nucypher.network.middleware import RestMiddleware
 
 
 class NodeConfiguration:
+
     _name = 'node'
+    _Character = NotImplemented
 
     DEFAULT_CONFIG_FILE_LOCATION = os.path.join(DEFAULT_CONFIG_ROOT, '{}.config'.format(_name))
-
-    _Character = NotImplemented
-    __parser = json.loads
-
     DEFAULT_OPERATING_MODE = 'decentralized'
+    NODE_SERIALIZER = binascii.hexlify
+    NODE_DESERIALIZER = binascii.unhexlify
 
+    __CONFIG_FILE_EXT = '.config'
+    __CONFIG_FILE_DESERIALIZER = json.loads
     __TEMP_CONFIGURATION_DIR_PREFIX = "nucypher-tmp-"
     __DEFAULT_NETWORK_MIDDLEWARE_CLASS = RestMiddleware
+    __DEFAULT_NODE_STORAGE = InMemoryNodeStorage
 
     __REGISTRY_NAME = 'contract_registry.json'
     REGISTRY_SOURCE = os.path.join(BASE_DIR, __REGISTRY_NAME)  # TODO: #461 Where will this be hosted?
@@ -69,7 +71,7 @@ class NodeConfiguration:
 
                  # Metadata
                  known_nodes: set = None,
-                 known_metadata_dir: str = None,
+                 node_storage: NodeStorage = None,
                  load_metadata: bool = True,
                  save_metadata: bool = True
 
@@ -80,7 +82,6 @@ class NodeConfiguration:
         # Known Nodes
         self.known_nodes_dir = constants.UNINITIALIZED_CONFIGURATION
         self.known_certificates_dir = known_certificates_dir or constants.UNINITIALIZED_CONFIGURATION
-        self.known_metadata_dir = known_metadata_dir or constants.UNINITIALIZED_CONFIGURATION
 
         # Keyring
         self.keyring = constants.UNINITIALIZED_CONFIGURATION
@@ -95,9 +96,14 @@ class NodeConfiguration:
         self.__temp = temp
         if self.__temp:
             self.__temp_dir = constants.UNINITIALIZED_CONFIGURATION
+            self.node_storage = InMemoryNodeStorage(federated_only=federated_only,
+                                                    character_class=self.__class__)
         else:
             self.config_root = config_root
             self.__temp_dir = constants.LIVE_CONFIGURATION
+            from nucypher.characters.lawful import Ursula  # TODO : Needs cleanup
+            self.node_storage = node_storage or self.__DEFAULT_NODE_STORAGE(federated_only=federated_only,
+                                                                            character_class=Ursula)
             self.__cache_runtime_filepaths()
         self.config_file_location = config_file_location
 
@@ -114,14 +120,13 @@ class NodeConfiguration:
             if checksum_address and not self.__temp:
                 self.read_keyring()
             self.network_middleware = network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS()
-
         else:
             #
             # Stranger
             #
             self.known_nodes_dir = constants.STRANGER_CONFIGURATION
             self.known_certificates_dir = constants.STRANGER_CONFIGURATION
-            self.known_metadata_dir = constants.STRANGER_CONFIGURATION
+            self.node_storage = constants.STRANGER_CONFIGURATION
             self.keyring_dir = constants.STRANGER_CONFIGURATION
             self.keyring = constants.STRANGER_CONFIGURATION
             self.network_middleware = constants.STRANGER_CONFIGURATION
@@ -142,10 +147,10 @@ class NodeConfiguration:
         # Auto-Initialization
         #
         if auto_initialize:
-            self.write(no_registry=not import_seed_registry or federated_only,
-                       wallet=auto_generate_keys and not federated_only,
-                       encrypting=auto_generate_keys,
-                       passphrase=passphrase)
+            self.initialize(no_registry=not import_seed_registry or federated_only,
+                            wallet=auto_generate_keys and not federated_only,
+                            encrypting=auto_generate_keys,
+                            passphrase=passphrase)
 
     def __call__(self, *args, **kwargs):
         return self.produce(*args, **kwargs)
@@ -158,7 +163,7 @@ class NodeConfiguration:
     def temp(self):
         return self.__temp
 
-    def produce(self, passphrase: str = None, **overrides) -> Character:
+    def produce(self, passphrase: str = None, **overrides):
         """Initialize a new character instance and return it"""
         if not self.temp:
             self.read_keyring()
@@ -169,17 +174,34 @@ class NodeConfiguration:
     @classmethod
     def from_configuration_file(cls, filepath, **overrides) -> 'NodeConfiguration':
         """Initialize a NodeConfiguration from a JSON file."""
+        from nucypher.config.storages import NodeStorage  # TODO: move
+        NODE_STORAGES = {storage_class._name: storage_class for storage_class in NodeStorage.__subclasses__()}
+
         with open(filepath, 'r') as file:
-            payload = cls.__parser(file.read())
+            payload = cls.__CONFIG_FILE_DESERIALIZER(file.read())
+
+        # Make NodeStorage
+        storage_payload = payload['node_storage']
+        storage_type = storage_payload[NodeStorage._TYPE_LABEL]
+        storage_class = NODE_STORAGES[storage_type]
+        node_storage = storage_class.from_payload(payload=storage_payload,
+                                                  serializer=cls.NODE_SERIALIZER,
+                                                  deserializer=cls.NODE_DESERIALIZER)
+        payload.update(dict(node_storage=node_storage))
         return cls(**{**payload, **overrides})
 
     def to_configuration_file(self, filepath: str = None) -> str:
         """Write the static_payload to a JSON file."""
         if filepath is None:
-            filename = '{}.config'.format(self._name.lower())
+            filename = '{}{}'.format(self._name.lower(), self.__CONFIG_FILE_EXT)
             filepath = os.path.join(self.config_root, filename)
+
+        payload = self.static_payload
+        # Save node connection data
+        payload.update(dict(node_storage=self.node_storage.payload()))
+
         with open(filepath, 'w') as config_file:
-            config_file.write(json.dumps(self.static_payload, indent=4))
+            config_file.write(json.dumps(payload, indent=4))
         return filepath
 
     def validate(self, config_root: str, no_registry=False) -> bool:
@@ -191,11 +213,11 @@ class NodeConfiguration:
         filepaths = self.runtime_filepaths
         if no_registry:
             del filepaths['registry_filepath']
+
         for field, path in filepaths.items():
             if not os.path.exists(path):
                 message = 'Missing configuration directory {}.'
                 raise NodeConfiguration.InvalidConfiguration(message.format(path))
-
         return True
 
     @property
@@ -207,14 +229,12 @@ class NodeConfiguration:
                     federated_only=self.federated_only,  # TODO: 466
                     checksum_address=self.checksum_address,
                     keyring_dir=self.keyring_dir,
+                    known_certificates_dir=self.known_certificates_dir,
 
                     # Behavior
                     learn_on_same_thread=self.learn_on_same_thread,
                     abort_on_learning_error=self.abort_on_learning_error,
                     start_learning_now=self.start_learning_now,
-
-                    known_certificates_dir=self.known_certificates_dir,
-                    known_metadata_dir=self.known_metadata_dir,
                     save_metadata=self.save_metadata
                 )
         return payload
@@ -223,10 +243,10 @@ class NodeConfiguration:
     def dynamic_payload(self, **overrides) -> dict:
         """Exported dynamic configuration values for initializing Ursula"""
         if self.load_metadata:
-            known_nodes = self.read_known_nodes(known_metadata_dir=self.known_metadata_dir)
-            self.known_nodes.update(known_nodes)
+            self.known_nodes.update(self.node_storage.all(federated_only=self.federated_only))
         payload = dict(network_middleware=self.network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS(),
                        known_nodes=self.known_nodes,
+                       node_storage=self.node_storage,
                        crypto_power_ups=self.derive_node_power_ups() or None)
         if overrides:
             self.log.debug("Overrides supplied to dynamic payload for {}".format(self.__class__.__name__))
@@ -237,9 +257,7 @@ class NodeConfiguration:
     def runtime_filepaths(self):
         filepaths = dict(config_root=self.config_root,
                          keyring_dir=self.keyring_dir,
-                         known_nodes_dir=self.known_nodes_dir,
                          known_certificates_dir=self.known_certificates_dir,
-                         known_metadata_dir=self.known_metadata_dir,
                          registry_filepath=self.registry_filepath)
         return filepaths
 
@@ -251,7 +269,6 @@ class NodeConfiguration:
                          keyring_dir=os.path.join(config_root, 'keyring'),
                          known_nodes_dir=known_nodes_dir,
                          known_certificates_dir=os.path.join(known_nodes_dir, 'certificates'),
-                         known_metadata_dir=os.path.join(known_nodes_dir, 'metadata'),
                          registry_filepath=os.path.join(config_root, NodeConfiguration.__REGISTRY_NAME))
         return filepaths
 
@@ -270,17 +287,17 @@ class NodeConfiguration:
                 power_ups.append(power_up)
         return power_ups
 
-    def write(self,
-              passphrase: str,
-              no_registry: bool = False,
-              wallet: bool = False,
-              encrypting: bool = False,
-              tls: bool = False,
-              host: str = None,
-              curve=None,
-              no_keys: bool = False
-              ) -> str:
-        """Write a new configuration to the disk"""
+    def initialize(self,
+                   passphrase: str,
+                   no_registry: bool = False,
+                   wallet: bool = False,
+                   encrypting: bool = False,
+                   tls: bool = False,
+                   host: str = None,
+                   curve=None,
+                   no_keys: bool = False
+                   ) -> str:
+        """Write a new configuration to the disk, and with the configured node store."""
 
         #
         # Create Config Root
@@ -308,7 +325,7 @@ class NodeConfiguration:
             os.mkdir(self.keyring_dir, mode=0o700)               # keyring
             os.mkdir(self.known_nodes_dir, mode=0o755)           # known_nodes
             os.mkdir(self.known_certificates_dir, mode=0o755)    # known_certs
-            os.mkdir(self.known_metadata_dir, mode=0o755)        # known_metadata
+            self.node_storage.initialize()  # TODO: default know dir
 
             if not self.temp and not no_keys:
                 # Keyring
@@ -335,24 +352,9 @@ class NodeConfiguration:
             self.validate(config_root=self.config_root, no_registry=no_registry or self.federated_only)
         return self.config_root
 
-    def read_known_nodes(self, known_metadata_dir=None) -> Set[Character]:
+    def read_known_nodes(self) -> set:
         """Read known nodes from metadata, and use them when producing a character"""
-        from nucypher.characters.lawful import Ursula
-
-        if known_metadata_dir is None:
-            known_metadata_dir = self.known_metadata_dir
-
-        glob_pattern = os.path.join(known_metadata_dir, '*.node')  # TODO: Use constant
-        metadata_paths = sorted(glob(glob_pattern), key=os.path.getctime)
-
-        self.log.info("Found {} known node metadata files at {}".format(len(metadata_paths), known_metadata_dir))
-        known_nodes = set()
-        for metadata_path in metadata_paths:
-            if self.checksum_address not in metadata_path:
-                node = Ursula.from_metadata_file(filepath=abspath(metadata_path), federated_only=self.federated_only)  # TODO: 466
-                known_nodes.add(node)
-
-        self.known_nodes.update(known_nodes)  # TODO: Use non-mutative approach?
+        known_nodes = self.node_storage.all()
         return known_nodes
 
     def read_keyring(self, *args, **kwargs):
@@ -378,6 +380,8 @@ class NodeConfiguration:
                                                 host=host,
                                                 curve=tls_curve,
                                                 keyring_root=self.keyring_dir)
+
+        # TODO: Operating mode switch
         if self.federated_only or not wallet:
             self.checksum_address = self.keyring.federated_address
         else:

nucypher/config/storages.py

@@ -0,0 +1,307 @@
+import binascii
+import os
+import tempfile
+from abc import abstractmethod, ABC
+from logging import getLogger
+
+import boto3 as boto3
+import shutil
+from botocore.errorfactory import ClientError
+from constant_sorrow import constants
+from typing import Callable
+
+from nucypher.config.constants import DEFAULT_CONFIG_ROOT
+
+
+class NodeStorage(ABC):
+
+    _name = NotImplemented
+    _TYPE_LABEL = 'storage_type'
+    NODE_SERIALIZER = binascii.hexlify
+    NODE_DESERIALIZER = binascii.unhexlify
+
+    class NodeStorageError(Exception):
+        pass
+
+    class UnknownNode(NodeStorageError):
+        pass
+
+    def __init__(self,
+                 character_class,
+                 federated_only: bool,  # TODO# 466
+                 serializer: Callable = NODE_SERIALIZER,
+                 deserializer: Callable = NODE_DESERIALIZER,
+                 ) -> None:
+
+        self.log = getLogger(self.__class__.__name__)
+        self.serializer = serializer
+        self.deserializer = deserializer
+        self.federated_only = federated_only
+        self.character_class = character_class
+
+    def __getitem__(self, item):
+        return self.get(checksum_address=item, federated_only=self.federated_only)
+
+    def __setitem__(self, key, value):
+        return self.save(node=value)
+
+    def __delitem__(self, key):
+        self.remove(checksum_address=key)
+
+    def __iter__(self):
+        return self.all(federated_only=self.federated_only)
+
+    @abstractmethod
+    def all(self, federated_only: bool) -> set:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get(self, checksum_address: str, federated_only: bool):
+        raise NotImplementedError
+
+    @abstractmethod
+    def save(self, node):
+        raise NotImplementedError
+
+    @abstractmethod
+    def remove(self, checksum_address: str) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def payload(self) -> str:
+        raise NotImplementedError
+
+    @classmethod
+    @abstractmethod
+    def from_payload(self, data: str, *args, **kwargs) -> 'NodeStorage':
+        raise NotImplementedError
+
+    @abstractmethod
+    def initialize(self):
+        raise NotImplementedError
+
+
+class InMemoryNodeStorage(NodeStorage):
+
+    _name = 'memory'
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.__known_nodes = dict()
+
+    def all(self, federated_only: bool) -> set:
+        return set(self.__known_nodes.values())
+
+    def get(self, checksum_address: str, federated_only: bool):
+        try:
+            return self.__known_nodes[checksum_address]
+        except KeyError:
+            raise self.UnknownNode
+
+    def save(self, node):
+        self.__known_nodes[node.checksum_public_address] = node
+        return True
+
+    def remove(self, checksum_address: str) -> bool:
+        del self.__known_nodes[checksum_address]
+        return True
+
+    def payload(self) -> dict:
+        payload = {self._TYPE_LABEL: self._name}
+        return payload
+
+    @classmethod
+    def from_payload(cls, payload: str, *args, **kwargs) -> 'InMemoryNodeStorage':
+        if payload[cls._TYPE_LABEL] != cls._name:
+            raise cls.NodeStorageError
+        return cls(*args, **kwargs)
+
+    def initialize(self) -> None:
+        self.__known_nodes = dict()
+
+
+class LocalFileBasedNodeStorage(NodeStorage):
+
+    _name = 'local'
+    __FILENAME_TEMPLATE = '{}.node'
+    __DEFAULT_DIR = os.path.join(DEFAULT_CONFIG_ROOT, 'known_nodes', 'metadata')
+
+    class NoNodeMetadataFileFound(FileNotFoundError, NodeStorage.UnknownNode):
+        pass
+
+    def __init__(self,
+                 known_metadata_dir: str = __DEFAULT_DIR,
+                 *args, **kwargs
+                 ) -> None:
+
+        super().__init__(*args, **kwargs)
+        self.log = getLogger(self.__class__.__name__)
+        self.known_metadata_dir = known_metadata_dir
+
+    def __generate_filepath(self, checksum_address: str) -> str:
+        metadata_path = os.path.join(self.known_metadata_dir, self.__FILENAME_TEMPLATE.format(checksum_address))
+        return metadata_path
+
+    def __read(self, filepath: str, federated_only: bool):
+        from nucypher.characters.lawful import Ursula
+        try:
+            with open(filepath, "rb") as seed_file:
+                seed_file.seek(0)
+                node_bytes = self.deserializer(seed_file.read())
+                node = Ursula.from_bytes(node_bytes, federated_only=federated_only)
+        except FileNotFoundError:
+            raise self.UnknownNode
+        return node
+
+    def __write(self, filepath: str, node):
+        with open(filepath, "wb") as f:
+            f.write(self.serializer(self.character_class.__bytes__(node)))
+        self.log.info("Wrote new node metadata to filesystem {}".format(filepath))
+        return filepath
+
+    def all(self, federated_only: bool) -> set:
+        filenames = os.listdir(self.known_metadata_dir)
+        self.log.info("Found {} known node metadata files at {}".format(len(filenames), self.known_metadata_dir))
+        known_nodes = set()
+        for filename in filenames:
+            metadata_path = os.path.join(self.known_metadata_dir, filename)
+            node = self.__read(filepath=metadata_path, federated_only=federated_only)   # TODO: 466
+            known_nodes.add(node)
+        return known_nodes
+
+    def get(self, checksum_address: str, federated_only: bool):
+        metadata_path = self.__generate_filepath(checksum_address=checksum_address)
+        node = self.__read(filepath=metadata_path, federated_only=federated_only)   # TODO: 466
+        return node
+
+    def save(self, node):
+        try:
+            filepath = self.__generate_filepath(checksum_address=node.checksum_public_address)
+        except AttributeError:
+            raise AttributeError("{} does not have a rest_interface attached".format(self))  # TODO.. eh?
+        self.__write(filepath=filepath, node=node)
+
+    def remove(self, checksum_address: str):
+        filepath = self.__generate_filepath(checksum_address=checksum_address)
+        self.log.debug("Delted {} from the filesystem".format(checksum_address))
+        return os.remove(filepath)
+
+    def payload(self) -> str:
+        payload = {
+            'storage_type': self._name,
+            'known_metadata_dir': self.known_metadata_dir
+        }
+        return payload
+
+    @classmethod
+    def from_payload(cls, payload: str, *args, **kwargs) -> 'LocalFileBasedNodeStorage':
+        storage_type = payload[cls._TYPE_LABEL]
+        if not storage_type == cls._name:
+            raise cls.NodeStorageError("Wrong storage type. got {}".format(storage_type))
+        return cls(known_metadata_dir=payload['known_metadata_dir'], *args, **kwargs)
+
+    def initialize(self):
+        try:
+            os.mkdir(self.known_metadata_dir, mode=0o755)  # known_metadata
+        except FileExistsError:
+            message = "There are pre-existing metadata files at {}".format(self.known_metadata_dir)
+            raise self.NodeStorageError(message)
+        except FileNotFoundError:
+            raise self.NodeStorageError("There is no existing configuration at {}".format(self.known_metadata_dir))
+
+
+class TemporaryFileBasedNodeStorage(LocalFileBasedNodeStorage):
+    _name = 'tmp'
+
+    def __init__(self, *args, **kwargs):
+        self.__temp_dir = constants.NO_STORAGE_AVAILIBLE
+        super().__init__(known_metadata_dir=self.__temp_dir, *args, **kwargs)
+
+    def __del__(self):
+        if not self.__temp_dir is constants.NO_STORAGE_AVAILIBLE:
+            shutil.rmtree(self.__temp_dir, ignore_errors=True)
+
+    def initialize(self):
+        self.__temp_dir = tempfile.mkdtemp(prefix="nucypher-tmp-nodes-")
+        self.known_metadata_dir = self.__temp_dir
+
+
+class S3NodeStorage(NodeStorage):
+    S3_ACL = 'private'  # Canned S3 Permissions
+
+    def __init__(self,
+                 bucket_name: str,
+                 s3_resource=None,
+                 *args, **kwargs) -> None:
+
+        super().__init__(*args, **kwargs)
+        self.__bucket_name = bucket_name
+        self.__s3client = boto3.client('s3')
+        self.__s3resource = s3_resource or boto3.resource('s3')
+        self.__bucket = constants.NO_STORAGE_AVAILIBLE
+
+    @property
+    def bucket(self):
+        return self.__bucket
+
+    @property
+    def bucket_name(self):
+        return self.__bucket_name
+
+    def __read(self, node_obj: str):
+        try:
+            node_object_metadata = node_obj.get()
+        except ClientError:
+            raise self.UnknownNode
+        node_bytes = self.deserializer(node_object_metadata['Body'].read())
+        node = self.character_class.from_bytes(node_bytes)
+        return node
+
+    def generate_presigned_url(self, checksum_address: str) -> str:
+        payload = {'Bucket': self.__bucket_name, 'Key': checksum_address}
+        url = self.__s3client.generate_presigned_url('get_object', payload, ExpiresIn=900)
+        return url
+
+    def all(self, federated_only: bool) -> set:
+        node_objs = self.__bucket.objects.all()
+        nodes = set()
+        for node_obj in node_objs:
+            node = self.__read(node_obj=node_obj)
+            nodes.add(node)
+        return nodes
+
+    def get(self, checksum_address: str, federated_only: bool):
+        node_obj = self.__bucket.Object(checksum_address)
+        node = self.__read(node_obj=node_obj)
+        return node
+
+    def save(self, node):
+        self.__s3client.put_object(Bucket=self.__bucket_name,
+                                   ACL=self.S3_ACL,
+                                   Key=node.checksum_public_address,
+                                   Body=self.serializer(bytes(node)))
+
+    def remove(self, checksum_address: str) -> bool:
+        node_obj = self.__bucket.Object(checksum_address)
+        response = node_obj.delete()
+        if response['ResponseMetadata']['HTTPStatusCode'] != 204:
+            raise self.NodeStorageError("S3 Storage failed to delete node {}".format(checksum_address))
+        return True
+
+    def payload(self) -> str:
+        payload = {
+            self._TYPE_LABEL: self._name,
+            'bucket_name': self.__bucket_name
+        }
+        return payload
+
+    @classmethod
+    def from_payload(cls, payload: str, *args, **kwargs):
+        return cls(bucket_name=payload['bucket_name'], *args, **kwargs)
+
+    def initialize(self):
+        self.__bucket = self.__s3resource.Bucket(self.__bucket_name)
+
+
+### Node Storage Registry ###
+NODE_STORAGES = {storage_class._name: storage_class for storage_class in NodeStorage.__subclasses__()}

nucypher/config/utils.py

@@ -1,72 +0,0 @@
-import configparser
-import os
-
-from typing import Union, Tuple
-
-from nucypher.config.keyring import NucypherKeyring
-from nucypher.config.node import NodeConfiguration
-
-
-def generate_local_wallet(keyring_root:str, passphrase: str) -> NucypherKeyring:
-    keyring = NucypherKeyring.generate(passphrase=passphrase,
-                                       keyring_root=keyring_root,
-                                       encrypting=False,
-                                       wallet=True)
-    return keyring
-
-
-def generate_account(w3, passphrase: str) -> NucypherKeyring:
-    address = w3.personal.newAccount(passphrase)
-    return address
-
-
-def check_config_permissions() -> bool:
-    rules = (
-        (os.name == 'nt' or os.getuid() != 0, 'Cannot run as root user.'),
-    )
-
-    for rule, failure_reason in rules:
-        if rule is not True:
-            raise Exception(failure_reason)
-    return True
-
-
-def validate_configuration_file(config=None,
-                                filepath: str = NodeConfiguration.DEFAULT_CONFIG_FILE_LOCATION,
-                                raise_on_failure: bool=False) -> Union[bool, Tuple[bool, tuple]]:
-
-    if config is None:
-        config = configparser.ConfigParser()
-        config.read(filepath)
-
-    if not config.sections():
-
-        raise NodeConfiguration.InvalidConfiguration("Empty configuration file")
-
-    required_sections = ("nucypher", "blockchain")
-
-    missing_sections = list()
-
-    try:
-        operating_mode = config["nucypher"]["mode"]
-    except KeyError:
-        raise NodeConfiguration.ConfigurationError("No operating mode configured")
-    else:
-        modes = ('federated', 'tester', 'decentralized', 'centralized')
-        if operating_mode not in modes:
-            missing_sections.append("mode")
-            if raise_on_failure is True:
-                raise NodeConfiguration.ConfigurationError("Invalid nucypher operating mode '{}'. Specify {}".format(operating_mode, modes))
-
-    for section in required_sections:
-        if section not in config.sections():
-            missing_sections.append(section)
-            if raise_on_failure is True:
-                raise NodeConfiguration.ConfigurationError("Invalid config file: missing section '{}'".format(section))
-
-    if len(missing_sections) > 0:
-        result = False, tuple(missing_sections)
-    else:
-        result = True, tuple()
-
-    return result

nucypher/network/server.py

@@ -156,7 +156,7 @@ class ProxyRESTRoutes:
                     self._suspicious_activity_tracker['vladimirs'].append(node)  # TODO: Maybe also record the bytes representation separately to disk?
                 except Exception as e:
                     self.log.critical(str(e))
-                    raise
+                    raise  # TODO
                 else:
                     self.log.info("Previously unknown node: {}".format(node.checksum_public_address))
                     self._node_recorder(node)

nucypher/utilities/sandbox/ursula.py

@@ -132,7 +132,7 @@ class UrsulaProcessProtocol(protocol.ProcessProtocol):
     def outReceived(self, data):
         print(data)
         if b'passphrase' in data:
-            self.transport.write(bytes(TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD, encoding='ascii'))
+            self.transport.__write(bytes(TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD, encoding='ascii'))
             self.transport.closeStdin()  # tell them we're done
 
     def errReceived(self, data):

tests/cli/test_config.py

@@ -7,43 +7,58 @@ from click.testing import CliRunner
 
 from cli.main import cli
 from nucypher.config.node import NodeConfiguration
+from nucypher.utilities.sandbox.constants import TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
+
+
+TEST_CUSTOM_INSTALLATION_PATH = '/tmp/nucypher-tmp-test-custom'
 
 
 @pytest.fixture(scope='function')
 def custom_filepath():
-    custom_filepath = '/tmp/nucypher-tmp-test-custom'
+    custom_filepath = TEST_CUSTOM_INSTALLATION_PATH
     yield custom_filepath
     with contextlib.suppress(FileNotFoundError):
-        shutil.rmtree(custom_filepath)
+        shutil.rmtree(custom_filepath, ignore_errors=True)
 
 
-@pytest.mark.skip()
-def test_initialize_configuration_directory(custom_filepath):
+def test_initialize_configuration_files_and_directories(custom_filepath):
     runner = CliRunner()
 
     # Use the system temporary storage area
-    result = runner.invoke(cli, ['--dev', 'configure', 'install', '--no-registry'], input='Y', catch_exceptions=False)
+    args = ['--dev', '--federated-only', 'configure', 'install', '--ursula', '--force']
+    result = runner.invoke(cli, args,
+                           input='{}\n{}'''.format(TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
+                                                   TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD),
+                           catch_exceptions=False)
     assert '/tmp' in result.output, "Configuration not in system temporary directory"
     assert NodeConfiguration._NodeConfiguration__TEMP_CONFIGURATION_DIR_PREFIX in result.output
     assert result.exit_code == 0
 
-    args = ['--config-root', custom_filepath, 'configure', 'install', '--no-registry']
-    result = runner.invoke(cli, args, input='Y', catch_exceptions=False)
-    assert '[y/N]' in result.output, "'configure init' did not prompt the user before attempting to write files"
-    assert '/tmp' in result.output, "Configuration not in system temporary directory"
+    # Use a custom local filepath
+    args = ['--config-root', custom_filepath, '--federated-only', 'configure', 'install', '--ursula', '--force']
+    result = runner.invoke(cli, args,
+                           input='{}\n{}'''.format(TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
+                                                   TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD),
+                           catch_exceptions=False)
+    assert TEST_CUSTOM_INSTALLATION_PATH in result.output, "Configuration not in system temporary directory"
     assert 'Created' in result.output
     assert custom_filepath in result.output
+    assert "'nucypher-cli ursula run'" in result.output
     assert result.exit_code == 0
     assert os.path.isdir(custom_filepath)
 
     # Ensure that there are not pre-existing configuration files at config_root
-    with pytest.raises(NodeConfiguration.ConfigurationError):
-        _result = runner.invoke(cli, args, input='Y', catch_exceptions=False)
+    _result = runner.invoke(cli, args,
+                           input='{}\n{}'''.format(TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD,
+                                                   TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD),
+                           catch_exceptions=False)
+    assert "There are existing configuration files" in _result.output
 
+    # Destroy / Uninstall
     args = ['--config-root', custom_filepath, 'configure', 'destroy']
     result = runner.invoke(cli, args, input='Y', catch_exceptions=False)
     assert '[y/N]' in result.output
-    assert '/tmp' in result.output, "Configuration not in system temporary directory"
+    assert TEST_CUSTOM_INSTALLATION_PATH in result.output, "Configuration not in system temporary directory"
     assert 'Deleted' in result.output
     assert custom_filepath in result.output
     assert result.exit_code == 0

tests/config/test_storages.py

@@ -0,0 +1,134 @@
+import boto3
+import pytest
+import requests
+from moto import mock_s3
+
+from nucypher.characters.lawful import Ursula
+from nucypher.config.storages import S3NodeStorage, InMemoryNodeStorage, TemporaryFileBasedNodeStorage, NodeStorage
+
+MOCK_S3_BUCKET_NAME = 'mock-bootnodes'
+S3_DOMAIN_NAME = 's3.amazonaws.com'
+
+
+@pytest.fixture(scope='function')
+def light_ursula(temp_dir_path):
+    node = Ursula(rest_host='127.0.0.1',
+                  rest_port=10151,
+                  federated_only=True)
+    return node
+
+
+@pytest.fixture(scope='function')
+def memory_node_storage():
+    _node_storage = InMemoryNodeStorage(character_class=Ursula, federated_only=True)
+    _node_storage.initialize()
+    return _node_storage
+
+
+@pytest.fixture(scope='function')
+def local_node_storage():
+    _node_storage = TemporaryFileBasedNodeStorage(character_class=Ursula, federated_only=True)
+    _node_storage.initialize()
+    return _node_storage
+
+
+@mock_s3
+def s3_node_storage_factory():
+    conn = boto3.resource('s3')
+    # We need to create the __bucket since this is all in Moto's 'virtual' AWS account
+    conn.create_bucket(Bucket=MOCK_S3_BUCKET_NAME, ACL=S3NodeStorage.S3_ACL)
+    _mock_storage = S3NodeStorage(bucket_name=MOCK_S3_BUCKET_NAME,
+                                  s3_resource=conn,
+                                  character_class=Ursula,
+                                  federated_only=True)
+    _mock_storage.initialize()
+    return _mock_storage
+
+
+class TestNodeStorageBackends:
+
+    def _read_and_write_to_storage(self, ursula, node_storage):
+        # Write Node
+        node_storage.save(node=ursula)
+
+        # Read Node
+        node_from_storage = node_storage.get(checksum_address=ursula.checksum_public_address,
+                                             federated_only=True)
+        assert ursula == node_from_storage, "Node storage {} failed".format(node_storage)
+
+        # Save more nodes
+        all_known_nodes = set()
+        for port in range(10152, 10251):
+            node = Ursula(rest_host='127.0.0.1', rest_port=port, federated_only=True)
+            node_storage.save(node=node)
+            all_known_nodes.add(node)
+
+        # Read all nodes from storage
+        all_stored_nodes = node_storage.all(federated_only=True)
+        all_known_nodes.add(ursula)
+        assert len(all_known_nodes) == len(all_stored_nodes)
+        assert all_stored_nodes == all_known_nodes
+
+        # Read random nodes
+        for i in range(3):
+            random_node = all_known_nodes.pop()
+            random_node_from_storage = node_storage.get(checksum_address=random_node.checksum_public_address,
+                                                        federated_only=True)
+            assert random_node.checksum_public_address == random_node_from_storage.checksum_public_address
+
+        return True
+
+    def _write_and_delete_nodes_in_storage(self, ursula, node_storage):
+        # Write Node
+        node_storage.save(node=ursula)
+
+        # Delete Node
+        node_storage.remove(checksum_address=ursula.checksum_public_address)
+
+        # Read Node
+        with pytest.raises(NodeStorage.UnknownNode):
+            _node_from_storage = node_storage.get(checksum_address=ursula.checksum_public_address,
+                                                  federated_only=True)
+
+        # Read all nodes from storage
+        all_stored_nodes = node_storage.all(federated_only=True)
+        assert all_stored_nodes == set()
+        return True
+
+
+    #
+    # Storage Backed Tests
+    #
+    @pytest.mark.parametrize("storage_factory", [
+        memory_node_storage,
+        local_node_storage,
+        s3_node_storage_factory
+    ])
+    @mock_s3
+    def test_delete_node_in_storage(self, light_ursula, storage_factory):
+        assert self._write_and_delete_nodes_in_storage(ursula=light_ursula, node_storage=storage_factory())
+
+    @pytest.mark.parametrize("storage_factory", [
+        memory_node_storage,
+        local_node_storage,
+        s3_node_storage_factory
+    ])
+    @mock_s3
+    def test_read_and_write_to_storage(self, light_ursula, storage_factory):
+        assert self._read_and_write_to_storage(ursula=light_ursula, node_storage=storage_factory())
+
+
+class TestS3NodeStorageDirect:
+
+    @mock_s3
+    def test_generate_presigned_url(self, light_ursula):
+        s3_node_storage = s3_node_storage_factory()
+        s3_node_storage.save(node=light_ursula)
+        presigned_url = s3_node_storage.generate_presigned_url(checksum_address=light_ursula.checksum_public_address)
+
+        assert S3_DOMAIN_NAME in presigned_url
+        assert MOCK_S3_BUCKET_NAME in presigned_url
+        assert light_ursula.checksum_public_address in presigned_url
+
+        moto_response = requests.get(presigned_url)
+        assert moto_response.status_code == 200

tests/fixtures.py

@@ -1,3 +1,4 @@
+import contextlib
 import datetime
 import os
 import re
@@ -37,10 +38,11 @@ from nucypher.utilities.sandbox.ursula import make_federated_ursulas, make_decen
 @pytest.fixture(scope="session", autouse=True)
 def cleanup():
     yield  # we've got a lot of men and women here...
-    for f in os.listdir(tempfile.tempdir):
-        if re.search(r'nucypher-*', f):
-            shutil.rmtree(os.path.join(tempfile.tempdir, f),
-                          ignore_errors=True)
+    with contextlib.suppress(FileNotFoundError):
+        for f in os.listdir(tempfile.tempdir):
+            if re.search(r'nucypher-*', f):
+                shutil.rmtree(os.path.join(tempfile.tempdir, f),
+                              ignore_errors=True)
 
 
 @pytest.fixture(scope="function")