Merge pull request #493 from jMyles/seed-node-learning

Seed node learning

cli/main.py

@@ -1090,21 +1090,21 @@ def status(config,
     color_index = {
         'self': 'yellow',
         'known': 'white',
-        'bootnode': 'blue'
+        'seednode': 'blue'
     }
     for node_type, color in color_index.items():
         click.secho('{0:<6} | '.format(node_type), fg=color, nl=False)
     click.echo('\n')
 
-    bootnode_addresses = list(bn.checksum_address for bn in BOOTNODES)
+    seednode_addresses = list(bn.checksum_address for bn in BOOTNODES)
     for node in known_nodes:
         row_template = "{} | {} | {}"
         node_type = 'known'
         if node.checksum_public_address == config.node_configuration.checksum_address:
             node_type = 'self'
             row_template += ' ({})'.format(node_type)
-        if node.checksum_public_address in bootnode_addresses:
+        if node.checksum_public_address in seednode_addresses:
-            node_type = 'bootnode'
+            node_type = 'seednode'
             row_template += ' ({})'.format(node_type)
         click.secho(row_template.format(node.checksum_public_address,
                                         node.rest_url(),
@@ -1196,8 +1196,8 @@ def ursula(config,
     config.operating_mode = "federated" if ursula_config.federated_only else "decentralized"
     click.secho("Running in {} mode".format(config.operating_mode), fg='blue')
 
-    # Bootnodes, Seeds, Known Nodes
+    # seednodes, Seeds, Known Nodes
-    ursula_config.load_bootnodes()
+    ursula_config.load_seednodes()
     quantity_known_nodes = len(ursula_config.known_nodes)
     if quantity_known_nodes > 0:
         click.secho("Loaded {} known nodes from storages".format(quantity_known_nodes, fg='blue'))

nucypher/characters/base.py

@@ -1,23 +1,24 @@
 import random
+import time
 from collections import defaultdict
 from collections import deque
 from contextlib import suppress
 from logging import Logger
 from logging import getLogger
 from tempfile import TemporaryDirectory
+from typing import Dict, ClassVar, Set
+from typing import Tuple
+from typing import Union, List
 
 import maya
 import requests
-import time
 from constant_sorrow import constants, default_constant_splitter
 from eth_keys import KeyAPI as EthKeyAPI
 from eth_utils import to_checksum_address, to_canonical_address
 from requests.exceptions import SSLError
-from twisted.internet import reactor
+from twisted.internet import reactor, defer
 from twisted.internet import task
-from typing import Dict, ClassVar, Set
+from twisted.internet.threads import deferToThread
-from typing import Tuple
-from typing import Union, List
 from umbral.keys import UmbralPublicKey
 from umbral.signing import Signature
 
@@ -29,7 +30,6 @@ from nucypher.crypto.powers import CryptoPower, SigningPower, EncryptingPower, N
 from nucypher.crypto.signing import signature_splitter, StrangerStamp, SignatureStamp
 from nucypher.network.middleware import RestMiddleware
 from nucypher.network.nodes import VerifiableNode
-from nucypher.network.server import TLSHostingPower
 
 
 class Learner:
@@ -63,12 +63,12 @@ class Learner:
                  known_nodes: tuple = None,
                  seed_nodes: Tuple[tuple] = None,
                  known_certificates_dir: str = None,
-                 node_storage = None,
+                 node_storage=None,
                  save_metadata: bool = False,
                  abort_on_learning_error: bool = False
                  ) -> None:
 
-        self.log = getLogger("characters")                       # type: Logger
+        self.log = getLogger("characters")  # type: Logger
 
         self.__common_name = common_name
         self.network_middleware = network_middleware
@@ -85,7 +85,8 @@ class Learner:
 
         # Read
         if node_storage is None:
-            node_storage = self.__DEFAULT_NODE_STORAGE(federated_only=self.federated_only,    #TODO: remove federated_only
+            node_storage = self.__DEFAULT_NODE_STORAGE(federated_only=self.federated_only,
+                                                       # TODO: remove federated_only
                                                        character_class=self.__class__)
 
         self.node_storage = node_storage
@@ -101,9 +102,9 @@ class Learner:
                 self.unresponsive_startup_nodes.append(node)
 
         self.teacher_nodes = deque()
-        self._current_teacher_node = None   # type: Teacher
+        self._current_teacher_node = None  # type: Teacher
         self._learning_task = task.LoopingCall(self.keep_learning_about_nodes)
-        self._learning_round = 0            # type: int
+        self._learning_round = 0  # type: int
         self._rounds_without_new_nodes = 0  # type: int
         self._seed_nodes = seed_nodes or []
 
@@ -120,40 +121,38 @@ class Learner:
                        retry_rate: int = 2,
                        timeout=3):
         """
-        Engage known nodes from storages and pre-fetch hardcoded bootnode certificates for node learning.
+        Engage known nodes from storages and pre-fetch hardcoded seednode certificates for node learning.
         """
-        def __attempt_bootnode_learning(bootnode, current_attempt=1):
-            self.log.debug("Loading Bootnode {}|{}:{}".format(bootnode.checksum_address, bootnode.rest_host, bootnode.rest_port))
-
-            try:
-                seed_node = self.network_middleware.learn_from_seednode(seednode_metadata=bootnode,
-                                                            timeout=timeout,
-                                                            accept_federated_only=self.federated_only)  # TODO: 466
-                self.remember_node(seed_node)
-            except RuntimeError:
-                if current_attempt == retry_attempts:
-                    message = "No Response from Bootnode {} after {} attempts"
-                    self.log.info(message.format(bootnode.rest_url, retry_attempts))
-                    return
-                unresponsive_seed_nodes.add(bootnode)
-                self.log.info("No Response from Bootnode {}. Retrying in {} seconds...".format(bootnode.rest_url, retry_rate))
-                time.sleep(retry_rate)
-                __attempt_bootnode_learning(bootnode=bootnode, current_attempt=current_attempt+1)
-            else:
-                self.log.info("Successfully learned from {}|{}:{}".format(bootnode.checksum_address, bootnode.rest_host, bootnode.rest_port))
-                if current_attempt > 1:
-                    unresponsive_seed_nodes.remove(bootnode)
-
-        for bootnode in self._seed_nodes:
-            __attempt_bootnode_learning(bootnode=bootnode)
-
         unresponsive_seed_nodes = set()
 
-        if len(unresponsive_seed_nodes) > 0:
+        def __attempt_seednode_learning(seednode_metadata, current_attempt=1):
-            self.log.info("No Bootnodes were availible after {} attempts".format(retry_attempts))
+            self.log.debug(
+                "Seeding from: {}|{}:{}".format(seednode_metadata.checksum_address,
+                                                seednode_metadata.rest_host,
+                                                seednode_metadata.rest_port))
+            seed_node = self.network_middleware.learn_about_seednode(seednode_metadata=seednode_metadata,
+                                                                    known_certs_dir=self.known_certificates_dir,
+                                                                    timeout=timeout,
+                                                                    accept_federated_only=self.federated_only)  # TODO: 466
+            if seed_node is False:
+                unresponsive_seed_nodes.add(seednode_metadata)
+            else:
+                self.remember_node(seed_node)
 
-        # if read_storages is True:
+        for seednode_metadata in self._seed_nodes:
-        #     self.read_known_nodes()
+            __attempt_seednode_learning(seednode_metadata=seednode_metadata)
+
+        if read_storages is True:
+            self.read_nodes_from_storage()
+
+        if not self.known_nodes:
+            self.log.warning("No seednodes were available after {} attempts".format(retry_attempts))
+            # TODO: Need some actual logic here for situation with no seed nodes (ie, maybe try again much later)
+
+    def read_nodes_from_storage(self) -> set:
+        stored_nodes = self.node_storage.all(federated_only=self.federated_only)  # TODO: 466
+        for node in stored_nodes:
+            self.remember_node(node)
 
     def remember_node(self, node, force_verification_check=False):
 
@@ -194,11 +193,17 @@ class Learner:
     def start_learning_loop(self, now=False):
         if self._learning_task.running:
             return False
-        else:
+        elif now:
             self.load_seednodes()
             d = self._learning_task.start(interval=self._SHORT_LEARNING_DELAY, now=now)
             d.addErrback(self.handle_learning_errors)
             return d
+        else:
+            seeder_deferred = deferToThread(self.load_seednodes)
+            learner_deferred = self._learning_task.start(interval=self._SHORT_LEARNING_DELAY, now=now)
+            seeder_deferred.addErrback(self.handle_learning_errors)
+            learner_deferred.addErrback(self.handle_learning_errors)
+            return defer.DeferredList([seeder_deferred, learner_deferred])
 
     def handle_learning_errors(self, *args, **kwargs):
         failure = args[0]
@@ -334,8 +339,9 @@ class Learner:
                 elif not self._learning_task.running:
                     raise self.NotEnoughTeachers("The learning loop is not running.  Start it with start_learning().")
                 else:
-                    raise self.NotEnoughTeachers("After {} seconds and {} rounds, didn't find these {} nodes: {}".format(
+                    raise self.NotEnoughTeachers(
-                        timeout, rounds_undertaken, len(still_unknown), still_unknown))
+                        "After {} seconds and {} rounds, didn't find these {} nodes: {}".format(
+                            timeout, rounds_undertaken, len(still_unknown), still_unknown))
 
             else:
                 time.sleep(.1)
@@ -411,7 +417,8 @@ class Learner:
         try:
 
             # TODO: Streamline path generation
-            certificate_filepath = current_teacher.get_certificate_filepath(certificates_dir=self.known_certificates_dir)
+            certificate_filepath = current_teacher.get_certificate_filepath(
+                certificates_dir=self.known_certificates_dir)
             response = self.network_middleware.get_nodes_via_rest(url=rest_url,
                                                                   nodes_i_need=self._node_ids_to_learn_about_immediately,
                                                                   announce_nodes=announce_nodes,
@@ -442,7 +449,8 @@ class Learner:
 
             try:
                 if eager:
-                    certificate_filepath = current_teacher.get_certificate_filepath(certificates_dir=certificate_filepath)
+                    certificate_filepath = current_teacher.get_certificate_filepath(
+                        certificates_dir=certificate_filepath)
                     node.verify_node(self.network_middleware,
                                      accept_federated_only=self.federated_only,  # TODO: 466
                                      certificate_filepath=certificate_filepath)
@@ -528,7 +536,7 @@ class Character(Learner):
 
         """
 
-        self.federated_only = federated_only           # type: bool
+        self.federated_only = federated_only  # type: bool
 
         #
         # Powers
@@ -538,7 +546,7 @@ class Character(Learner):
         crypto_power_ups = crypto_power_ups or list()  # type: list
 
         if crypto_power:
-            self._crypto_power = crypto_power          # type: CryptoPower
+            self._crypto_power = crypto_power  # type: CryptoPower
         elif crypto_power_ups:
             self._crypto_power = CryptoPower(power_ups=crypto_power_ups)
         else:
@@ -552,7 +560,7 @@ class Character(Learner):
                 self.blockchain = blockchain or Blockchain.connect()
 
             self.keyring_dir = keyring_dir  # type: str
-            self.treasure_maps = {}         # type: dict
+            self.treasure_maps = {}  # type: dict
             self.network_middleware = network_middleware or RestMiddleware()
 
             #
@@ -560,7 +568,7 @@ class Character(Learner):
             #
             try:
                 signing_power = self._crypto_power.power_ups(SigningPower)  # type: SigningPower
-                self._stamp = signing_power.get_signature_stamp()           # type: SignatureStamp
+                self._stamp = signing_power.get_signature_stamp()  # type: SignatureStamp
             except NoSigningPower:
                 self._stamp = constants.NO_SIGNING_POWER
 

nucypher/config/node.py

@@ -355,11 +355,6 @@ class NodeConfiguration:
             self.validate(config_root=self.config_root, no_registry=no_registry or self.federated_only)
         return self.config_root
 
-    def read_known_nodes(self) -> set:
-        """Read known nodes from metadata, and use them when producing a character"""
-        known_nodes = self.node_storage.all(federated_only=self.federated_only)
-        return known_nodes
-
     def read_keyring(self, *args, **kwargs):
         if self.checksum_address is None:
             raise self.ConfigurationError("No account specified to unlock keyring")

nucypher/network/middleware.py

@@ -1,9 +1,17 @@
+import os
+import socket
+import ssl
+import time
+
 import requests
-
 from bytestring_splitter import BytestringSplitter, VariableLengthBytestring
-
+from cryptography import x509
-from umbral.fragments import CapsuleFrag
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.serialization import Encoding
 from twisted.logger import Logger
+from umbral.fragments import CapsuleFrag
+
+from nucypher.config.keyring import _write_tls_certificate
 
 
 class RestMiddleware:
@@ -19,16 +27,64 @@ class RestMiddleware:
             raise RuntimeError("Bad response: {}".format(response.content))
         return response
 
-    def _get_certificate(self, hostname, port):
+    def learn_about_seednode(self, seednode_metadata, known_certs_dir, timeout=3, accept_federated_only=False):
-        bootnode_certificate = ssl.get_server_certificate(hostname, port)
+        from nucypher.characters.lawful import Ursula
-        certificate = x509.load_pem_x509_certificate(bootnode_certificate.encode(),
+        # Pre-fetch certificate
+        self.log.info("Fetching seednode {} TLS certificate".format(seednode_metadata.checksum_address))
+
+        # TODO: Utilize timeout.
+        certificate, filepath = self._get_certificate(checksum_address=seednode_metadata.checksum_address,
+                                                      hostname=seednode_metadata.rest_host,
+                                                      port=seednode_metadata.rest_port,
+                                                      certs_dir=known_certs_dir,
+                                                      timeout=timeout)
+
+        if certificate is False:
+            return False
+
+        potential_seed_node = Ursula.from_rest_url(self,
+                                                   seednode_metadata.rest_host,
+                                                   seednode_metadata.rest_port,
+                                                   certificate_filepath=filepath,
+                                                   federated_only=True)  # TODO: 466
+
+        if not seednode_metadata.checksum_address == potential_seed_node.checksum_public_address:
+            raise potential_seed_node.SuspiciousActivity(
+                "This seed node has a different wallet address: {} (was hoping for {}).  Are you sure this is a seed node?".format(
+                    potential_seed_node.checksum_public_address,
+                    seednode_metadata.checksum_address))
+        try:
+            potential_seed_node.verify_node(self,
+                                            accept_federated_only=accept_federated_only,
+                                            certificate_filepath=filepath)
+        except potential_seed_node.InvalidNode:
+            raise  # TODO: What if our seed node fails verification?
+
+        return potential_seed_node
+
+    def _get_certificate(self, checksum_address, certs_dir, hostname, port,
+                         timeout=3, retry_attempts: int=3, retry_rate: int = 2,):
+        socket.setdefaulttimeout(timeout)  # Set Socket Timeout
+        current_attempt = 0
+        try:
+            seednode_certificate = ssl.get_server_certificate(addr=(hostname, port))
+        except socket.timeout:
+            if current_attempt == retry_attempts:
+                message = "No Response from seednode {} after {} attempts"
+                self.log.info(message.format(checksum_address, retry_attempts))
+                return False, False
+            self.log.info(
+                "No Response from seednode {}. Retrying in {} seconds...".format(checksum_address, retry_rate))
+            time.sleep(retry_rate)
+
+        certificate = x509.load_pem_x509_certificate(seednode_certificate.encode(),
                                                      backend=default_backend())
         # Write certificate
-        filename = '{}.{}'.format(bootnode.checksum_address, Encoding.PEM.name.lower())
+        filename = '{}.{}'.format(checksum_address, Encoding.PEM.name.lower())
-        certificate_filepath = os.path.join(self.known_certificates_dir, filename)
+        certificate_filepath = os.path.join(certs_dir, filename)
         _write_tls_certificate(certificate=certificate, full_filepath=certificate_filepath, force=True)
-        self.log.info("Saved bootnode {} TLS certificate".format(bootnode.checksum_address))
+        self.log.info("Saved seednode {} TLS certificate".format(checksum_address))
-
+        return certificate, certificate_filepath
 
     def enact_policy(self, ursula, id, payload):
         response = requests.post('https://{}/kFrag/{}'.format(ursula.rest_interface, id.hex()), payload,

nucypher/utilities/sandbox/middleware.py

@@ -1,9 +1,4 @@
-from urllib.parse import urlparse
-
 from apistar import TestClient
-from cryptography import x509
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.serialization import Encoding
 
 from nucypher.characters.lawful import Ursula
 from nucypher.network.middleware import RestMiddleware
@@ -37,31 +32,8 @@ class MockRestMiddleware(RestMiddleware):
             raise RuntimeError(
                 "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port))
 
-    def learn_from_seednode(self, seednode_metadata, timeout=3, accept_federated_only=False):
+    def _get_certificate(self, checksum_address, certs_dir, hostname, port, timeout=3, retry_attempts: int = 3,
-        # Pre-fetch certificate
+                         retry_rate: int = 2, ):
-        self.log.info("Fetching bootnode {} TLS certificate".format(seednode_metadata.checksum_address))
-
-        certificate, filepath = self._get_certificate(seednode_metadata.rest_host, seednode_metadata.rest_port)
-
-        potential_seed_node = Ursula.from_rest_url(self,
-                                                   seednode_metadata.rest_host,
-                                                   seednode_metadata.rest_port,
-                                                   certificate_filepath=filepath,
-                                                   federated_only=True)  # TODO: 466
-
-        if not seednode_metadata.checksum_address == potential_seed_node.checksum_public_address:
-            raise potential_seed_node.SuspiciousActivity(
-                "This seed node has a different wallet address: {} (was hoping for {}).  Are you sure this is a seed node?".format(
-                    potential_seed_node.checksum_public_address,
-                    bootnode.checksum_address))
-        try:
-            potential_seed_node.verify_node(self, accept_federated_only=accept_federated_only)
-        except potential_seed_node.InvalidNode:
-            raise  # TODO: What if our seed node fails verification?
-
-        return potential_seed_node
-
-    def _get_certificate(self, hostname, port):
         ursula = self._get_ursula_by_port(port)
         return ursula.certificate, ursula.certificate_filepath
 

nucypher/utilities/sandbox/ursula.py

@@ -143,21 +143,21 @@ class UrsulaCommandProtocol(LineReceiver):
         color_index = {
             'self': 'yellow',
             'known': 'white',
-            'bootnode': 'blue'
+            'seednode': 'blue'
         }
         for node_type, color in color_index.items():
             click.secho('{0:<6} | '.format(node_type), fg=color, nl=False)
         click.echo('\n')
 
-        bootnode_addresses = list(bn.checksum_address for bn in BOOTNODES)
+        seednode_addresses = list(bn.checksum_address for bn in BOOTNODES)
         for address, node in known_nodes.items():
             row_template = "{} | {} | {}"
             node_type = 'known'
             if node.checksum_public_address == self.ursula.checksum_public_address:
                 node_type = 'self'
                 row_template += ' ({})'.format(node_type)
-            if node.checksum_public_address in bootnode_addresses:
+            if node.checksum_public_address in seednode_addresses:
-                node_type = 'bootnode'
+                node_type = 'seednode'
                 row_template += ' ({})'.format(node_type)
             click.secho(row_template.format(node.checksum_public_address,
                                             node.rest_url(),

tests/config/test_firstula_circumstances.py

@@ -1,6 +1,13 @@
 from functools import partial
 
+import maya
+import pytest
+import pytest_twisted
+from twisted.internet.threads import deferToThread
+
+from nucypher.network.middleware import RestMiddleware
 from nucypher.utilities.sandbox.ursula import make_federated_ursulas
+from cryptography.hazmat.primitives import serialization
 
 
 def test_proper_seed_node_instantiation(ursula_federated_test_config):
@@ -10,9 +17,44 @@ def test_proper_seed_node_instantiation(ursula_federated_test_config):
                                   know_each_other=False)
 
     firstula = lonely_ursula_maker().pop()
-
+    firstula_as_seed_node = firstula.seed_node_metadata()
-    any_other_ursula = lonely_ursula_maker(seed_nodes=[firstula.seed_node_metadata()]).pop()
+    any_other_ursula = lonely_ursula_maker(seed_nodes=[firstula_as_seed_node]).pop()
 
     assert not any_other_ursula.known_nodes
-    any_other_ursula.start_learning_loop()
+    any_other_ursula.start_learning_loop(now=True)
     assert firstula in any_other_ursula.known_nodes.values()
+
+
+@pytest_twisted.inlineCallbacks
+def test_get_cert_from_running_seed_node(ursula_federated_test_config):
+    lonely_ursula_maker = partial(make_federated_ursulas,
+                                  ursula_config=ursula_federated_test_config,
+                                  quantity=1,
+                                  know_each_other=False)
+    firstula = lonely_ursula_maker().pop()
+    node_deployer = firstula.get_deployer()
+
+    node_deployer.addServices()
+    node_deployer.catalogServers(node_deployer.hendrix)
+    node_deployer.start()
+
+    certificate_as_deployed = node_deployer.cert.to_cryptography()
+
+    firstula_as_seed_node = firstula.seed_node_metadata()
+    any_other_ursula = lonely_ursula_maker(seed_nodes=[firstula_as_seed_node],
+                                           network_middleware=RestMiddleware()).pop()
+    assert not any_other_ursula.known_nodes
+
+    def start_lonely_learning_loop():
+        any_other_ursula.start_learning_loop()
+        start = maya.now()
+        while not firstula in any_other_ursula.known_nodes.values():
+            passed = maya.now() - start
+            if passed.seconds > 2:
+                pytest.fail("Didn't find the seed node.")
+
+    yield deferToThread(start_lonely_learning_loop)
+    assert firstula in any_other_ursula.known_nodes.values()
+
+    certificate_as_learned = list(any_other_ursula.known_nodes.values())[0].certificate
+    assert certificate_as_learned == certificate_as_deployed

tests/config/test_storages.py

@@ -6,7 +6,7 @@ from moto import mock_s3
 from nucypher.characters.lawful import Ursula
 from nucypher.config.storages import S3NodeStorage, InMemoryNodeStorage, TemporaryFileBasedNodeStorage, NodeStorage
 
-MOCK_S3_BUCKET_NAME = 'mock-bootnodes'
+MOCK_S3_BUCKET_NAME = 'mock-seednodes'
 S3_DOMAIN_NAME = 's3.amazonaws.com'
 
 

tests/network/test_network_upgrade.py

@@ -1,6 +1,4 @@
 import os
-
-import pytest
 import pytest_twisted
 import requests
 from cryptography.hazmat.primitives import serialization
@@ -40,13 +38,7 @@ def test_federated_nodes_connect_via_tls_and_verify(ursula_federated_test_config
 
     try:
         with open("test-cert", "wb") as f:
-            # f.write(cert.tbs_certificate_bytes.hex())
             f.write(cert_bytes)
         yield threads.deferToThread(check_node_with_cert, node, "test-cert")
     finally:
         os.remove("test-cert")
-
-
-@pytest.mark.skip(reason="To be implemented")
-def test_node_metadata_contains_proper_cert():
-    pass

tests/network/test_node_storage.py

@@ -1,6 +1,34 @@
+import maya
 import pytest
+import pytest_twisted
+from twisted.internet.threads import deferToThread
+
+from nucypher.utilities.sandbox.ursula import make_federated_ursulas
 
 
-@pytest.mark.skip("To be implemented.")
+@pytest_twisted.inlineCallbacks
-def test_eager_learn_from_teacher():
+def test_one_node_stores_a_bunch_of_others(federated_ursulas, ursula_federated_test_config):
-    assert False
+    the_chosen_seednode = list(federated_ursulas)[2]
+    seed_node = the_chosen_seednode.seed_node_metadata()
+    newcomer = make_federated_ursulas(
+        ursula_config=ursula_federated_test_config,
+        quantity=1,
+        know_each_other=False,
+        save_metadata=True,
+        seed_nodes=[seed_node]).pop()
+
+    assert not newcomer.known_nodes
+
+    def start_lonely_learning_loop():
+        newcomer.start_learning_loop()
+        start = maya.now()
+        # Loop until the_chosen_seednode is in storage.
+        while not the_chosen_seednode in newcomer.node_storage.all(federated_only=True):
+            passed = maya.now() - start
+            if passed.seconds > 2:
+                pytest.fail("Didn't find the seed node.")
+
+    yield deferToThread(start_lonely_learning_loop)
+
+    # The known_nodes are all saved in storage (and no others have been saved)
+    assert list(newcomer.known_nodes.values()) == list(newcomer.node_storage.all(True))