Merge pull request #2657 from piotr-roslaniec/teacher-fallback#2481

Fallback chain of teacher nodes

newsfragments/2657.feature.rst

@@ -0,0 +1 @@
+Adds "https://closest-seed.nucypher.network" and "https://mainnet.nucypher.network" as a fallback teacher nodes for mainnet.

nucypher/characters/lawful.py

@@ -94,7 +94,7 @@ from nucypher.datastore.datastore import DatastoreTransactionError, RecordNotFou
 from nucypher.datastore.queries import find_expired_policies, find_expired_treasure_maps
 from nucypher.network.exceptions import NodeSeemsToBeDown
 from nucypher.network.middleware import RestMiddleware
-from nucypher.network.nodes import NodeSprout, Teacher
+from nucypher.network.nodes import NodeSprout, TEACHER_NODES, Teacher
 from nucypher.network.protocols import InterfaceInfo, parse_node_uri
 from nucypher.network.server import ProxyRESTServer, TLSHostingPower, make_rest_app
 from nucypher.network.trackers import AvailabilityTracker
@@ -1459,7 +1459,7 @@ class Ursula(Teacher, Character, Worker):
     def seednode_for_network(cls, network: str) -> 'Ursula':
         """Returns a default seednode ursula for a given network."""
         try:
-            url = RestMiddleware.TEACHER_NODES[network][0]
+            url = TEACHER_NODES[network][0]
         except KeyError:
             raise ValueError(f'"{network}" is not a known network.')
         except IndexError:

nucypher/network/middleware.py

@@ -145,12 +145,6 @@ class RestMiddleware:
 
     _client_class = NucypherMiddlewareClient
 
-    TEACHER_NODES = {
-        NetworksInventory.MAINNET: ('https://seeds.nucypher.network:9151',),
-        NetworksInventory.LYNX: ('https://lynx.nucypher.network:9151',),
-        NetworksInventory.IBEX: ('https://ibex.nucypher.network:9151',),
-    }
-
     class UnexpectedResponse(Exception):
         def __init__(self, message, status, *args, **kwargs):
             super().__init__(message, *args, **kwargs)

nucypher/network/nodes.py

@@ -16,49 +16,64 @@ along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
 """
 
 import contextlib
-import datetime
 import time
 from collections import defaultdict, deque
 from contextlib import suppress
 from queue import Queue
-from typing import Iterable, List
-from typing import Set, Tuple, Union
+from typing import Iterable, List, Set, Tuple, Union
 
 import maya
 import requests
+from bytestring_splitter import (
+    BytestringSplitter,
+    PartiallyKwargifiedBytes,
+    VariableLengthBytestring
+)
+from constant_sorrow import constant_or_bytes
+from constant_sorrow.constants import (
+    CERTIFICATE_NOT_SAVED,
+    FLEET_STATES_MATCH,
+    NOT_SIGNED,
+    NO_KNOWN_NODES,
+    NO_STORAGE_AVAILABLE,
+    RELAX,
+    UNKNOWN_VERSION
+)
 from cryptography.x509 import Certificate
 from eth_utils import to_checksum_address
 from requests.exceptions import SSLError
 from twisted.internet import reactor, task
 from twisted.internet.defer import Deferred
+from umbral.signing import Signature
 
-import nucypher
-from bytestring_splitter import BytestringSplitter, BytestringSplittingError, PartiallyKwargifiedBytes, \
-    VariableLengthBytestring
-from constant_sorrow import constant_or_bytes
-from constant_sorrow.constants import (CERTIFICATE_NOT_SAVED, FLEET_STATES_MATCH, NOT_SIGNED,
-                                       NO_KNOWN_NODES, NO_STORAGE_AVAILIBLE, UNKNOWN_FLEET_STATE, UNKNOWN_VERSION,
-                                       RELAX)
 from nucypher.acumen.nicknames import Nickname
 from nucypher.acumen.perception import FleetSensor
 from nucypher.blockchain.economics import EconomicsFactory
 from nucypher.blockchain.eth.agents import ContractAgency, StakingEscrowAgent
 from nucypher.blockchain.eth.constants import NULL_ADDRESS
+from nucypher.blockchain.eth.networks import NetworksInventory
 from nucypher.blockchain.eth.registry import BaseContractRegistry
 from nucypher.config.constants import SeednodeMetadata
 from nucypher.config.storages import ForgetfulNodeStorage
-from nucypher.crypto.api import recover_address_eip_191, verify_eip_191, InvalidNodeCertificate
+from nucypher.crypto.api import InvalidNodeCertificate, recover_address_eip_191, verify_eip_191
 from nucypher.crypto.kits import UmbralMessageKit
-from nucypher.crypto.powers import DecryptingPower, NoSigningPower, SigningPower, TransactingPower
+from nucypher.crypto.powers import DecryptingPower, NoSigningPower, SigningPower
 from nucypher.crypto.signing import signature_splitter
 from nucypher.network import LEARNING_LOOP_VERSION
 from nucypher.network.exceptions import NodeSeemsToBeDown
 from nucypher.network.middleware import RestMiddleware
 from nucypher.network.protocols import SuspiciousActivity
-from nucypher.network.server import TLSHostingPower
 from nucypher.utilities.logging import Logger
-from umbral.signing import Signature
 
+TEACHER_NODES = {
+    NetworksInventory.MAINNET: (
+        'https://closest-seed.nucypher.network:9151',
+        'https://seeds.nucypher.network',
+        'https://mainnet.nucypher.network:9151',
+    ),
+    NetworksInventory.LYNX: ('https://lynx.nucypher.network:9151',),
+    NetworksInventory.IBEX: ('https://ibex.nucypher.network:9151',),
+}
 
 class NodeSprout(PartiallyKwargifiedBytes):
     """
@@ -241,7 +256,7 @@ class Learner:
         if not node_storage:
             node_storage = self.__DEFAULT_NODE_STORAGE(federated_only=self.federated_only)
         self.node_storage = node_storage
-        if save_metadata and node_storage is NO_STORAGE_AVAILIBLE:
+        if save_metadata and node_storage is NO_STORAGE_AVAILABLE:
             raise ValueError("Cannot save nodes without a configured node storage")
 
         from nucypher.characters.lawful import Ursula
@@ -301,7 +316,7 @@ class Learner:
         discovered = []
 
         if self.domain:
-            canonical_sage_uris = self.network_middleware.TEACHER_NODES.get(self.domain, ())
+            canonical_sage_uris = TEACHER_NODES.get(self.domain, ())
 
             for uri in canonical_sage_uris:
                 try:

nucypher/utilities/networking.py

@@ -114,18 +114,16 @@ def get_external_ip_from_default_teacher(network: str,
                                          log: Logger = IP_DETECTION_LOGGER
                                          ) -> Union[str, None]:
 
-    # Prevents circular import
+    # Prevents circular imports
     from nucypher.characters.lawful import Ursula
+    from nucypher.network.nodes import TEACHER_NODES
 
     if federated_only and registry:
         raise ValueError('Federated mode must not be true if registry is provided.')
+
     base_error = 'Cannot determine IP using default teacher'
-    try:
-        top_teacher_url = RestMiddleware.TEACHER_NODES[network][0]
-    except IndexError:
-        log.debug(f'{base_error}: No teacher available for network "{network}".')
-        return
-    except KeyError:
+
+    if network not in TEACHER_NODES:
         log.debug(f'{base_error}: Unknown network "{network}".')
         return
 
@@ -136,17 +134,26 @@ def get_external_ip_from_default_teacher(network: str,
     Ursula.set_federated_mode(federated_only)
     #####
 
-    try:
-        teacher = Ursula.from_teacher_uri(teacher_uri=top_teacher_url,
-                                          federated_only=federated_only,
-                                          min_stake=0)  # TODO: Handle customized min stake here.
-    except NodeSeemsToBeDown:
-        # Teacher is unreachable.  Move on.
+    external_ip = None
+    for teacher_uri in TEACHER_NODES[network]:
+        try:
+            teacher = Ursula.from_teacher_uri(teacher_uri=teacher_uri,
+                                              federated_only=federated_only,
+                                              min_stake=0)  # TODO: Handle customized min stake here.
+            # TODO: Pass registry here to verify stake (not essential here since it's a hardcoded node)
+            external_ip = _request_from_node(teacher=teacher)
+            # Found a reachable teacher, return from loop
+            if external_ip:
+                break
+        except NodeSeemsToBeDown:
+            # Teacher is unreachable, try next one
+            continue
+
+    if not external_ip:
+        log.debug(f'{base_error}: No teacher available for network "{network}".')
         return
 
-    # TODO: Pass registry here to verify stake (not essential here since it's a hardcoded node)
-    result = _request_from_node(teacher=teacher)
-    return result
+    return external_ip
 
 
 def get_external_ip_from_known_nodes(known_nodes: FleetSensor,

tests/fixtures.py

@@ -15,28 +15,28 @@ You should have received a copy of the GNU Affero General Public License
 along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-
-import json
-
 import contextlib
-import maya
+import json
 import os
-import pytest
 import random
 import shutil
 import tempfile
-from click.testing import CliRunner
 from datetime import datetime, timedelta
-from eth_utils import to_checksum_address
 from functools import partial
-from typing import Tuple, Callable
+from typing import Callable, Tuple
+
+import maya
+import pytest
+from click.testing import CliRunner
+from constant_sorrow.constants import (FULL, INIT)
+from eth_utils import to_checksum_address
 from web3 import Web3
 from web3.contract import Contract
 from web3.types import TxReceipt
 
 from nucypher.blockchain.economics import BaseEconomics, StandardTokenEconomics
 from nucypher.blockchain.eth.actors import StakeHolder, Staker
-from nucypher.blockchain.eth.agents import NucypherTokenAgent, PolicyManagerAgent, StakingEscrowAgent, ContractAgency
+from nucypher.blockchain.eth.agents import ContractAgency, NucypherTokenAgent
 from nucypher.blockchain.eth.deployers import (
     AdjudicatorDeployer,
     NucypherTokenDeployer,
@@ -60,6 +60,7 @@ from nucypher.config.characters import (
 from nucypher.config.constants import TEMPORARY_DOMAIN
 from nucypher.crypto.powers import TransactingPower
 from nucypher.datastore import datastore
+from nucypher.network.nodes import TEACHER_NODES
 from nucypher.utilities.logging import GlobalLoggerSettings, Logger
 from tests.constants import (
     BASE_TEMP_DIR,
@@ -100,14 +101,8 @@ from tests.utils.config import (
 )
 from tests.utils.middleware import MockRestMiddleware, MockRestMiddlewareForLargeFleetTests
 from tests.utils.policy import generate_random_label
-from tests.utils.ursula import (
-    MOCK_URSULA_STARTING_PORT,
-    make_decentralized_ursulas,
-    make_federated_ursulas,
-    MOCK_KNOWN_URSULAS_CACHE,
-    _mock_ursula_reencrypts
-)
-from constant_sorrow.constants import (FULL, INIT)
+from tests.utils.ursula import (MOCK_KNOWN_URSULAS_CACHE, MOCK_URSULA_STARTING_PORT, _mock_ursula_reencrypts,
+                                make_decentralized_ursulas, make_federated_ursulas)
 
 test_logger = Logger("test-logger")
 
@@ -1035,3 +1030,9 @@ def stakeholder_configuration_file_location(custom_filepath):
     _configuration_file_location = os.path.join(MOCK_CUSTOM_INSTALLATION_PATH,
                                                 StakeHolderConfiguration.generate_filename())
     return _configuration_file_location
+
+
+@pytest.fixture(autouse=True)
+def mock_teacher_nodes(mocker):
+    mock_nodes = tuple(u.rest_url() for u in MOCK_KNOWN_URSULAS_CACHE.values())[0:2]
+    mocker.patch.dict(TEACHER_NODES, {TEMPORARY_DOMAIN: mock_nodes}, clear=True)

tests/integration/learning/test_domains.py

@@ -16,7 +16,9 @@
 """
 
 from nucypher.acumen.perception import FleetSensor
+from nucypher.characters.lawful import Ursula
 from nucypher.config.storages import LocalFileBasedNodeStorage
+from nucypher.network.nodes import TEACHER_NODES
 
 
 def test_learner_learns_about_domains_separately(lonely_ursula_maker, caplog):
@@ -118,3 +120,47 @@ def test_learner_ignores_stored_nodes_from_other_domains(lonely_ursula_maker, tm
     other_staker._current_teacher_node = learner
     other_staker.learn_from_teacher_node()  # And once it did, the node from the wrong domain spread.
     assert pest not in other_staker.known_nodes  # But not anymore.
+
+
+def test_learner_with_empty_storage_uses_fallback_nodes(lonely_ursula_maker, mocker):
+    domain = "learner-domain"
+    mocker.patch.dict(TEACHER_NODES, {domain: ("teacher-uri",)}, clear=True)
+
+    # Create a learner and a teacher
+    learner, teacher = lonely_ursula_maker(domain=domain, quantity=2, save_metadata=False)
+    mocker.patch.object(Ursula, 'from_teacher_uri', return_value=teacher)
+
+    # Since there are no nodes in local node storage, the learner should only learn about the teacher
+    learner.learn_from_teacher_node()
+    assert set(learner.known_nodes) == {teacher}
+
+
+def test_learner_uses_both_nodes_from_storage_and_fallback_nodes(lonely_ursula_maker, tmpdir, mocker):
+    domain = "learner-domain"
+    mocker.patch.dict(TEACHER_NODES, {domain: ("teacher-uri",)}, clear=True)
+
+    # Create a local file-based node storage
+    root = tmpdir.mkdir("known_nodes")
+    metadata = root.mkdir("metadata")
+    certs = root.mkdir("certs")
+    node_storage = LocalFileBasedNodeStorage(federated_only=True,
+                                             metadata_dir=metadata,
+                                             certificates_dir=certs,
+                                             storage_root=root)
+
+    # Create some nodes and persist them to local storage
+    other_nodes = lonely_ursula_maker(domain=domain,
+                                      node_storage=node_storage,
+                                      know_each_other=True,
+                                      quantity=3,
+                                      save_metadata=True)
+
+    # Create a teacher and a learner using existing node storage
+    learner, teacher = lonely_ursula_maker(domain=domain, node_storage=node_storage, quantity=2)
+    mocker.patch.object(Ursula, 'from_teacher_uri', return_value=teacher)
+
+    # The learner should learn about all nodes
+    learner.learn_from_teacher_node()
+    all_nodes = {teacher}
+    all_nodes.update(other_nodes)
+    assert set(learner.known_nodes) == all_nodes

tests/metrics/grant_availability.py

@@ -18,6 +18,8 @@
 """
 from pathlib import Path
 
+from nucypher.network.nodes import TEACHER_NODES
+
 """
 WARNING: This script makes automatic transactions.
 Do not use this script unless you know what you
@@ -67,7 +69,7 @@ except KeyError:
 # Alice Configuration
 DOMAIN: str = 'mainnet'  # ibex
 DEFAULT_SEEDNODE_URIS: List[str] = [
-    *RestMiddleware.TEACHER_NODES[DOMAIN],
+    *TEACHER_NODES[DOMAIN],
 ]
 INSECURE_PASSWORD: str = "METRICS_INSECURE_DEVELOPMENT_PASSWORD"
 TEMP_ALICE_DIR: str = Path('/', 'tmp', 'grant-metrics')

tests/unit/test_external_ip_utilities.py

@@ -20,7 +20,8 @@ import pytest
 from nucypher.acumen.perception import FleetSensor
 from nucypher.characters.lawful import Ursula
 from nucypher.network.exceptions import NodeSeemsToBeDown
-from nucypher.network.middleware import RestMiddleware, NucypherMiddlewareClient
+from nucypher.network.middleware import NucypherMiddlewareClient
+from nucypher.network.nodes import TEACHER_NODES
 from nucypher.utilities.networking import (
     determine_external_ip_address,
     get_external_ip_from_centralized_source,
@@ -31,7 +32,6 @@ from nucypher.utilities.networking import (
 )
 from tests.constants import MOCK_IP_ADDRESS
 
-
 MOCK_NETWORK = 'holodeck'
 
 
@@ -79,7 +79,7 @@ def mock_client(mocker):
 @pytest.fixture(autouse=True)
 def mock_default_teachers(mocker):
     teachers = {MOCK_NETWORK: (MOCK_IP_ADDRESS, )}
-    mocker.patch.dict(RestMiddleware.TEACHER_NODES, teachers)
+    mocker.patch.dict(TEACHER_NODES, teachers, clear=True)
 
 
 def test_get_external_ip_from_centralized_source(mock_requests):
@@ -140,7 +140,7 @@ def test_get_external_ip_from_known_nodes_client(mocker, mock_client):
 
     # Setup HTTP Client
     mocker.patch.object(Ursula, 'from_teacher_uri', return_value=Dummy('0xdeadpork'))
-    teacher_uri = RestMiddleware.TEACHER_NODES[MOCK_NETWORK][0]
+    teacher_uri = TEACHER_NODES[MOCK_NETWORK][0]
 
     get_external_ip_from_known_nodes(known_nodes=sensor, sample_size=sample_size)
     assert mock_client.call_count == 1  # first node responded
@@ -161,7 +161,7 @@ def test_get_external_ip_default_teacher_unreachable(mocker):
 def test_get_external_ip_from_default_teacher(mocker, mock_client, mock_requests):
 
     mock_client.return_value = Dummy.GoodResponse
-    teacher_uri = RestMiddleware.TEACHER_NODES[MOCK_NETWORK][0]
+    teacher_uri = TEACHER_NODES[MOCK_NETWORK][0]
     mocker.patch.object(Ursula, 'from_teacher_uri', return_value=Dummy('0xdeadbeef'))
 
     # "Success"

tests/utils/middleware.py

@@ -96,16 +96,6 @@ class MockRestMiddleware(RestMiddleware):
     class NotEnoughMockUrsulas(Ursula.NotEnoughUrsulas):
         pass
 
-    class TEACHER_NODES:
-
-        @classmethod
-        def get(_cls, item, _default):
-            if item is TEMPORARY_DOMAIN:
-                nodes = tuple(u.rest_url() for u in MOCK_KNOWN_URSULAS_CACHE.values())[0:2]
-            else:
-                nodes = tuple()
-            return nodes
-
     def get_certificate(self, host, port, timeout=3, retry_attempts: int = 3, retry_rate: int = 2,
                         current_attempt: int = 0):
         ursula = self.client._get_ursula_by_port(port)