Pre-init bootnode connection retries

nucypher/config/node.py

@@ -1,7 +1,9 @@
 import binascii
 import json
 import os
+import socket
 import ssl
+import time
 from json import JSONDecodeError
 from logging import getLogger
 from tempfile import TemporaryDirectory
@@ -434,40 +436,75 @@ class NodeConfiguration:
         self.log.info("Successfully wrote registry to {}".format(output_filepath))
         return output_filepath
 
-    def get_bootnodes(self, read_storages: bool = True):
+    def __learn_from_bootnode(self, bootnode):
+        parsed_url = urlparse(bootnode.rest_url)
+
+        # Pre-fetch certificate
+        self.log.info("Fetching bootnode {} TLS certificate".format(bootnode.checksum_address))
+        bootnode_certificate = ssl.get_server_certificate((parsed_url.hostname, parsed_url.port))
+        certificate = x509.load_pem_x509_certificate(bootnode_certificate.encode(),
+                                                     backend=default_backend())
+
+        # Write certificate
+        filename = '{}.{}'.format(bootnode.checksum_address, Encoding.PEM.name.lower())
+        certificate_filepath = os.path.join(self.known_certificates_dir, filename)
+        _write_tls_certificate(certificate=certificate, full_filepath=certificate_filepath, force=True)
+        self.log.info("Saved bootnode {} TLS certificate".format(bootnode.checksum_address))
+
+        # Learn from Bootnode
+        response = self.network_middleware.get_nodes_via_rest(url=parsed_url.netloc,
+                                                              certificate_filepath=certificate_filepath)
+        self.log.info("Retrieved bootnode data from {}".format(bootnode.checksum_address))
+
+        if response.status_code != 200:
+            raise RuntimeError("Bad response from bootnode {}".format(bootnode.rest_url))
+
+        signature, nodes = signature_splitter(response.content, return_remainder=True)
+        node_list = Ursula.batch_from_bytes(nodes, federated_only=self.federated_only)  # TODO: 466
+        self.log.debug("Learned from Bootnode {}|{}".format(bootnode.checksum_address, parsed_url.geturl()))
+
+        for node in node_list:
+            self.known_nodes.add(node)
+
+        return node_list
+
+    def load_bootnodes(self,
+                       read_storages: bool = True,
+                       load_seed_nodes: bool = True,
+                       retry_attempts: int = 3,
+                       retry_rate: int = 2,
+                       timeout=3):
         """
-        Engage known nodes from storages and pre-fetch hardcoded bootnode certificates.
+        Engage known nodes from storages and pre-fetch hardcoded bootnode certificates for node learning.
         """
-        self.log.debug("Loading bootnodes")
+        if load_seed_nodes is True:
+            socket.setdefaulttimeout(timeout)  # Set Socket Timeout
 
-        # Hardcoded Bootnodes
-        for bootnode in BOOTNODES:
-            parsed_url = urlparse(bootnode.rest_url)
-            bootnode_certificate = ssl.get_server_certificate((parsed_url.hostname, parsed_url.port))
-            certificate = x509.load_pem_x509_certificate(bootnode_certificate.encode(), backend=default_backend())
-            filename = '{}.{}'.format(bootnode.checksum_address, Encoding.PEM.name.lower())
-            certificate_filepath = os.path.join(self.known_certificates_dir, filename)
-            _write_tls_certificate(certificate=certificate, full_filepath=certificate_filepath, force=True)
+            unresponsive_seed_nodes = set()
 
-            unresponsive_nodes = set()
-            try:
-                response = self.network_middleware.get_nodes_via_rest(url=parsed_url.netloc,
-                                                                      certificate_filepath=certificate_filepath)
-            except requests.exceptions.ConnectionError as e:
-                unresponsive_nodes.add(bootnode)
-                self.log.info("No Response from Bootnode {}".format(bootnode.rest_url))
-                raise e
+            def __attempt_bootnode_learning(bootnode, current_attempt=1):
+                self.log.debug("Loading Bootnode {}|{}".format(bootnode.checksum_address, bootnode.rest_url))
 
-            else:
-                if response.status_code != 200:
-                    raise RuntimeError("Bad response from bootnode {}".format(bootnode.rest_url))
-                signature, nodes = signature_splitter(response.content, return_remainder=True)
-                node_list = Ursula.batch_from_bytes(nodes, federated_only=self.federated_only)  # TODO: 466
+                try:
+                    self.__learn_from_bootnode(bootnode=bootnode)
+                except socket.timeout:
+                    if current_attempt == retry_attempts:
+                        message = "No Response from Bootnode {} after {} attempts"
+                        self.log.info(message.format(bootnode.rest_url, retry_attempts))
+                        return
+                    unresponsive_seed_nodes.add(bootnode)
+                    self.log.info("No Response from Bootnode {}. Retrying in {} seconds...".format(bootnode.rest_url, retry_rate))
+                    time.sleep(retry_rate)
+                    __attempt_bootnode_learning(bootnode=bootnode, current_attempt=current_attempt+1)
+                else:
+                    self.log.info("Successfully learned from bootnode {}".format(bootnode.rest_url))
+                    if current_attempt > 1:
+                        unresponsive_seed_nodes.remove(bootnode)
 
-            for node in node_list:
-                self.known_nodes.add(node)
-                self.log.debug("Connected to Bootnode {}|{}".format(bootnode.checksum_address, parsed_url.geturl()))
+            for bootnode in BOOTNODES:
+                __attempt_bootnode_learning(bootnode=bootnode)
+            if len(unresponsive_seed_nodes) > 0:
+                self.log.info("No Bootnodes were availible after {} attempts".format(retry_attempts))
 
-        # Get Known Nodes
         if read_storages is True:
             self.read_known_nodes()