Extricate DHT usage from CLI; egin to move node storages closer to characters.

cli/main.py

@@ -121,12 +121,6 @@ class NucypherClickConfig:
         uri_template = "http://{}:{}"
         return uri_template.format(host, port)
 
-    @property
-    def dht_uri(self):
-        host, port = self.payload['dht_host'], self.payload['dht_port']
-        uri_template = "http://{}:{}"
-        return uri_template.format(host, port)
-
 
 uses_config = click.make_pass_decorator(NucypherClickConfig, ensure=True)
 
@@ -376,7 +370,6 @@ def stake(config, action, address, index, value, duration):
         raise NotImplementedError
 
 
-
 @cli.command()
 @click.argument('action')
 @click.option('--federated-only', is_flag=True)
@@ -493,15 +486,15 @@ def simulate(config, action, nodes, federated_only):
             # Parse ursula parameters
             #
 
-            rest_port, dht_port = sim_port_number, sim_port_number + 100
+            rest_port = sim_port_number
             db_name = 'sim-{}'.format(rest_port)
 
             cli_exec = os.path.join(BASE_DIR, 'cli', 'main.py')
             python_exec = 'python'
 
             proc_params = '''
-            python3 {} run_ursula --host {} --rest-port {} --dht-port {} --db-name {}
-            '''.format(python_exec, cli_exec, localhost, rest_port, dht_port, db_name).split()
+            python3 {} run_ursula --host {} --rest-port {} --db-name {}
+            '''.format(python_exec, cli_exec, localhost, rest_port, db_name).split()
 
             if federated_only:
                 click.echo("Setting federated operating mode")
@@ -537,12 +530,10 @@ def simulate(config, action, nodes, federated_only):
             # Start with some basic status data, then build on it
 
             rest_uri = "http://{}:{}".format(localhost, rest_port)
-            dht_uri = "http://{}:{}".format(localhost, dht_port)
 
-            sim_data = "Started simulated Ursula | ReST {} | DHT {} ".format(rest_uri, dht_uri)
+            sim_data = "Started simulated Ursula | ReST {}".format(rest_uri)
             rest_uri = "{host}:{port}".format(host=localhost, port=str(sim_port_number))
-            dht_uri = '{host}:{port}'.format(host=localhost, port=dht_port)
-            sim_data.format(rest_uri, dht_uri)
+            sim_data.format(rest_uri)
 
             if not federated_only:
                 stake_infos = tuple(config.miner_agent.get_all_stakes(miner_address=sim_address))
@@ -684,24 +675,23 @@ def status(config, provider, contracts, network):
 
 @cli.command()
 @click.option('--federated-only', is_flag=True, default=False)
-@click.option('--teacher-uri', type=str)
-@click.option('--seed-node', is_flag=True, default=False)
+@click.option('--dev', is_flag=True, default=False)
 @click.option('--rest-host', type=str, default='localhost')
 @click.option('--rest-port', type=int, default=DEFAULT_REST_PORT)
 @click.option('--db-name', type=str, default=DEFAULT_DB_NAME)
 @click.option('--checksum-address', type=str)
-@click.option('--data-dir', type=click.Path(), default=DEFAULT_CONFIG_ROOT)
+@click.option('--teacher-uri', type=str)
+@click.option('--node-dir', type=click.Path(), default=DEFAULT_CONFIG_ROOT)
 @click.option('--config-file', type=click.Path(), default=DEFAULT_INI_FILEPATH)
 def run_ursula(rest_port,
                rest_host,
-               dht_port,
                db_name,
                teacher_uri,
                checksum_address,
                federated_only,
-               seed_node,
-               data_dir,
-               config_file) -> None:
+               node_dir,
+               config_file,
+               dev) -> None:
     """
 
     The following procedure is required to "spin-up" an Ursula node.
@@ -717,19 +707,18 @@ def run_ursula(rest_port,
     but can be overridden (mostly for testing purposes) with inline cli options.
 
     """
+    if dev is True:
+        pass
 
-    other_nodes = collect_stored_nodes(federated_only=federated_only)  # 1. Collect known nodes
+    other_nodes = collect_stored_nodes()              # 1. Collect known nodes
 
     ursula_params = dict(federated_only=federated_only,
                          known_nodes=other_nodes,
                          rest_host=rest_host,
                          rest_port=rest_port,
-                         dht_port=dht_port,
                          db_name=db_name,
                          checksum_address=checksum_address)
 
-    asyncio.set_event_loop(asyncio.new_event_loop())  # 2. Init DHT async loop
-
     if teacher_uri:
         host, port = teacher_uri.split(':')
         teacher = Ursula(rest_host=host,

nucypher/characters.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 import random
 from collections import OrderedDict, defaultdict
 from collections import deque
@@ -6,7 +7,7 @@ from contextlib import suppress
 from logging import Logger
 from logging import getLogger
 
-import kademlia
+import binascii
 import maya
 import requests
 import time
@@ -28,7 +29,7 @@ from typing import Union, List
 from nucypher.blockchain.eth.actors import PolicyAuthor, Miner, only_me
 from nucypher.blockchain.eth.agents import MinerAgent
 from nucypher.blockchain.eth.constants import datetime_to_period
-from nucypher.config.constants import DEFAULT_INI_FILEPATH
+from nucypher.config.constants import DEFAULT_INI_FILEPATH, DEFAULT_KNOWN_NODE_DIR
 from nucypher.config.parsers import parse_ursula_config, parse_alice_config, \
     parse_character_config
 from nucypher.crypto.api import keccak_digest, encrypt_and_sign
@@ -41,7 +42,7 @@ from nucypher.keystore.keypairs import HostingKeypair
 from nucypher.network.middleware import RestMiddleware
 from nucypher.network.nodes import VerifiableNode
 from nucypher.network.protocols import InterfaceInfo
-from nucypher.network.server import NucypherDHTServer, NucypherSeedOnlyDHTServer, ProxyRESTServer, TLSHostingPower, \
+from nucypher.network.server import NucypherDHTServer, ProxyRESTServer, TLSHostingPower, \
     ProxyRESTRoutes
 from umbral.keys import UmbralPublicKey
 from umbral.signing import Signature
@@ -795,7 +796,6 @@ class Alice(Character, PolicyAuthor):
 
 
 class Bob(Character):
-    _dht_server_class = NucypherSeedOnlyDHTServer
     _default_crypto_powerups = [SigningPower, EncryptingPower]
 
     def __init__(self, *args, **kwargs):
@@ -1311,35 +1311,18 @@ class Ursula(Character, VerifiableNode, Miner):
     # Utilities
     #
 
-    def attach_dht_server(self,
-                          ksize: int = 20,
-                          alpha: int = 3,
-                          node_id=None,
-                          storage=None,
-                          *args, **kwargs) -> None:
+    def write_node_metadata(self, node_metadata_dir: str = DEFAULT_KNOWN_NODE_DIR) -> str:
 
-        node_id = node_id or bytes(
-            self.canonical_public_address)  # Ursula can still "mine" wallets until she gets a DHT ID she wants.  Does that matter?  #136
-        # TODO What do we actually want the node ID to be?  Do we want to verify it somehow?  136
-        super().attach_dht_server(ksize=ksize, id=digest(node_id), alpha=alpha, storage=storage)
+        try:
+            filename = "node-metadata-{}".format(self.rest_information()[0].port)
+        except AttributeError:
+            raise AttributeError("{} does not have a rest_interface attached".format(self))
 
-    def dht_listen(self):
-        if self.dht_interface is constants.NO_INTERFACE:
-            raise TypeError("This node does not have a DHT interface configured.")
-        return self.dht_server.listen(self.dht_interface.port,
-                                      self.dht_interface.host)
+        metadata_filepath = os.path.join(node_metadata_dir, filename)
+        with open(metadata_filepath, "w") as f:
+            f.write(bytes(self).hex())
 
-    def publish_dht_information(self):
-        # TODO: Simplify or wholesale deprecate this.  337
-        if not self.dht_interface:
-            raise RuntimeError("Must listen before publishing interface information.")
-
-        ursula_id = self.canonical_public_address
-        interface_value = constants.BYTESTRING_IS_URSULA_IFACE_INFO + bytes(self)
-        setter = self.dht_server.set(key=ursula_id, value=interface_value)
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(setter)
-        return interface_value
+        return metadata_filepath
 
     def work_orders(self, bob=None):
         """

nucypher/config/metadata.py

@@ -1,63 +0,0 @@
-import binascii
-import os
-from glob import glob
-from os.path import abspath
-
-from nucypher.characters import Ursula
-from nucypher.config.constants import DEFAULT_SEED_NODE_DIR, DEFAULT_KNOWN_NODE_DIR, DEFAULT_CONFIG_ROOT
-
-
-def read_node_metadata(filepath: str, federated_only=False) -> Ursula:
-
-    """Init one ursula from node storage file"""
-
-    with open(filepath, "r") as seed_file:
-        seed_file.seek(0)
-        node_bytes = binascii.unhexlify(seed_file.read())
-
-        node = Ursula.from_bytes(node_bytes, federated_only=federated_only)
-        return node
-
-
-def write_node_metadata(node: Ursula,
-                        seed_node: bool = False,
-                        node_metadata_dir: str = DEFAULT_KNOWN_NODE_DIR) -> str:
-
-    try:
-        filename = "node-metadata-{}".format(node.rest_information()[0].port)
-    except AttributeError:
-        raise AttributeError("{} does not have a rest_interface attached".format(node))
-
-    node_type = 'known' if not seed_node else 'seed'
-    metadata_filepath = os.path.join(node_metadata_dir, '{}_nodes'.format(node_type), filename)
-
-    with open(metadata_filepath, "w") as f:
-        f.write(bytes(node).hex())
-
-    return metadata_filepath
-
-
-def collect_stored_nodes(config_root=DEFAULT_CONFIG_ROOT,
-                         seed_nodes_dirs=None,
-                         known_nodes_dirs: list=None,
-                         federated_only=False) -> tuple:
-    """Collect stored node data from multiple sources and aggregate them into known node sets"""
-
-    if known_nodes_dirs is None:
-        known_nodes_dirs = [DEFAULT_KNOWN_NODE_DIR]
-
-    if seed_nodes_dirs is None:
-        seed_nodes_dirs = [DEFAULT_SEED_NODE_DIR]
-
-    # TODO: use seed bucket?
-    _all_node_dirs = known_nodes_dirs + seed_nodes_dirs
-
-    glob_pattern = os.path.join(config_root, '*', 'node-metadata-*',)
-    metadata_paths = sorted(glob(glob_pattern), key=os.path.getctime)
-
-    nodes = list()
-    for metadata_path in metadata_paths:
-        node = read_node_metadata(filepath=abspath(metadata_path), federated_only=federated_only)
-        nodes.append(node)
-
-    return tuple(nodes)

nucypher/network/server.py

@@ -21,7 +21,6 @@ from nucypher.crypto.powers import SigningPower, KeyPairBasedPower, PowerUpError
 from nucypher.keystore.keypairs import HostingKeypair
 from nucypher.keystore.threading import ThreadedSession
 from nucypher.network.protocols import NucypherSeedOnlyProtocol, NucypherHashProtocol, InterfaceInfo
-from nucypher.network.storage import SeedOnlyStorage
 
 
 class NucypherDHTServer(Server):
@@ -93,14 +92,6 @@ class NucypherDHTServer(Server):
         return await self.set_digest(key, value)
 
 
-class NucypherSeedOnlyDHTServer(NucypherDHTServer):
-    protocol_class = NucypherSeedOnlyProtocol
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.storage = SeedOnlyStorage()
-
-
 class ProxyRESTServer:
     log = getLogger("characters")
 

nucypher/network/storage.py

@@ -1,7 +0,0 @@
-from kademlia.storage import ForgetfulStorage
-
-
-class SeedOnlyStorage(ForgetfulStorage):
-
-    def __setitem__(self, key, value):
-        pass