From 4de9b91d2af02737b1f3af2d77436973a9bbe989 Mon Sep 17 00:00:00 2001 From: Bogdan Opanchuk Date: Mon, 12 Oct 2020 16:09:48 -0700 Subject: [PATCH] Refactor FleetSensor --- .../worker/include/check_running_ursula.yml | 2 +- nucypher/acumen/nicknames.py | 7 +- nucypher/acumen/perception.py | 402 +++++++++++++----- nucypher/characters/base.py | 2 + nucypher/characters/lawful.py | 10 +- nucypher/cli/painting/nodes.py | 17 +- nucypher/cli/processes.py | 3 +- nucypher/network/nodes.py | 174 ++++---- nucypher/network/server.py | 28 +- nucypher/network/templates/basic_status.mako | 75 ++-- nucypher/network/trackers.py | 2 +- nucypher/utilities/prometheus/collector.py | 2 +- .../characters/test_ursula_web_status.py | 8 +- .../acceptance/network/test_network_actors.py | 6 +- tests/fixtures.py | 6 +- .../characters/test_ursula_startup.py | 3 +- tests/integration/learning/test_domains.py | 2 +- .../learning/test_firstula_circumstances.py | 2 +- .../integration/learning/test_fleet_state.py | 51 ++- .../integration/network/test_failure_modes.py | 4 +- tests/mock/performance_mocks.py | 3 +- tests/unit/test_external_ip_utilities.py | 81 ++-- 22 files changed, 528 insertions(+), 362 deletions(-) diff --git a/deploy/ansible/worker/include/check_running_ursula.yml b/deploy/ansible/worker/include/check_running_ursula.yml index 0860f64ae..91ed2c6d3 100644 --- a/deploy/ansible/worker/include/check_running_ursula.yml +++ b/deploy/ansible/worker/include/check_running_ursula.yml @@ -53,7 +53,7 @@ debug: msg: "local nickname: {{host_nickname}}\n - {% if serving and 'json' in status_data %}nickname: {{status_data.json.nickname}}\n + {% if serving and 'json' in status_data %}nickname: {{status_data.json.nickname.text}}\n staker address: {{status_data.json.staker_address}}\n worker address: {{status_data.json.worker_address}}\n rest url: https://{{status_data.json.rest_url}}\n diff --git a/nucypher/acumen/nicknames.py b/nucypher/acumen/nicknames.py index 1344c5450..c8e65ffe7 100644 --- a/nucypher/acumen/nicknames.py +++ b/nucypher/acumen/nicknames.py @@ -74,7 +74,7 @@ class NicknameCharacter: self.color_hex = color_hex self._text = color_name + " " + _SYMBOLS[symbol] - def payload(self): + def to_json(self): return dict(symbol=self.symbol, color_name=self.color_name, color_hex=self.color_hex) @@ -103,8 +103,9 @@ class Nickname: self.icon = "[" + "".join(character.symbol for character in characters) + "]" self.characters = characters - def payload(self): - return [character.payload() for character in self.characters] + def to_json(self): + return dict(text=self._text, + characters=[character.to_json() for character in self.characters]) def __str__(self): return self._text diff --git a/nucypher/acumen/perception.py b/nucypher/acumen/perception.py index 696258ee9..8c7548c5b 100644 --- a/nucypher/acumen/perception.py +++ b/nucypher/acumen/perception.py @@ -16,7 +16,9 @@ """ import binascii +import itertools import random +import weakref import maya @@ -30,45 +32,137 @@ from nucypher.crypto.api import keccak_digest from nucypher.utilities.logging import Logger -class FleetSensor: - """ - A representation of a fleet of NuCypher nodes. - """ - _checksum = NO_KNOWN_NODES.bool_value(False) - _nickname = NO_KNOWN_NODES - _tracking = False - most_recent_node_change = NO_KNOWN_NODES - snapshot_splitter = BytestringSplitter(32, 4) - log = Logger("Learning") - FleetState = namedtuple("FleetState", ("nickname", "icon", "nodes", "updated", "checksum")) +class BaseFleetState: - def __init__(self, domain: str): - self.domain = domain - self.additional_nodes_to_track = [] - self.updated = maya.now() - self._nodes = OrderedDict() - self._marked = defaultdict(list) # Beginning of bucketing. - self.states = OrderedDict() - - def __setitem__(self, checksum_address, node_or_sprout): - if node_or_sprout.domain == self.domain: - self._nodes[checksum_address] = node_or_sprout - - if self._tracking: - self.log.info("Updating fleet state after saving node {}".format(node_or_sprout)) - self.record_fleet_state() + def __str__(self): + if len(self) != 0: + # TODO: draw the icon in color, similarly to the web version? + return '{checksum} ⇀{nickname}↽ {icon}'.format(icon=self.nickname.icon, + nickname=self.nickname, + checksum=self.checksum[:7]) else: - msg = f"Rejected node {node_or_sprout} because its domain is '{node_or_sprout.domain}' but we're only tracking '{self.domain}'" - self.log.warn(msg) + return 'No Known Nodes' + + +class ArchivedFleetState(BaseFleetState): + + def __init__(self, checksum, nickname, timestamp, population): + self.checksum = checksum + self.nickname = nickname + self.timestamp = timestamp + self.population = population + + def to_json(self): + return dict(checksum=self.checksum, + nickname=self.nickname.to_json() if self.nickname is not None else 'NO_KNOWN_NODES', + timestamp=self.timestamp.rfc2822(), + population=self.population) + + +# Assumptions we're based on: +# - Every supplied node object, after its constructor has finished, +# has a ``.checksum_address`` and ``bytes()`` (metadata) +# - checksum address or metadata does not change for the same Python object +# - ``this_node`` (the owner of FleetSensor) may not have a checksum address initially +# (when the constructor is first called), but will have one at the time of the first +# `record_fleet_state()` call. This applies to its metadata as well. +# - The metadata of ``this_node`` **can** change. +# - For the purposes of the fleet state, nodes with different metadata are considered different, +# even if they have the same checksum address. + +class FleetState(BaseFleetState): + + @classmethod + def new(cls, this_node=None): + this_node_ref = weakref.ref(this_node) if this_node is not None else None + # Using empty checksum so that JSON library is not confused. + # Plus, we do need some checksum anyway. It's a legitimate state after all. + return cls(checksum=keccak_digest(b"").hex(), + nodes={}, + this_node_ref=this_node_ref, + this_node_metadata=None) + + def __init__(self, checksum, nodes, this_node_ref, this_node_metadata): + self.checksum = checksum + self.nickname = None if checksum is None else Nickname.from_seed(checksum, length=1) + self._nodes = nodes + self.timestamp = maya.now() + self._this_node_ref = this_node_ref + self._this_node_metadata = this_node_metadata + + def archived(self): + return ArchivedFleetState(checksum=self.checksum, + nickname=self.nickname, + timestamp=self.timestamp, + population=self.population) + + def with_updated_nodes(self, new_nodes, marked_nodes): + + # Checking if the node already has a checksum address + # (it may be created later during the constructor) + # or if it mutated since the last check. + if self._this_node_ref is not None and getattr(self._this_node_ref(), 'finished_initializing', False): + this_node = self._this_node_ref() + this_node_metadata = bytes(this_node) + this_node_changed = self._this_node_metadata != this_node_metadata + this_node_list = [this_node] + else: + this_node_metadata = self._this_node_metadata + this_node_changed = False + this_node_list = [] + + new_nodes = {checksum_address: node for checksum_address, node in new_nodes.items() + if checksum_address not in marked_nodes} + + remote_nodes_updated = any( + (checksum_address not in self._nodes or bytes(self._nodes[checksum_address]) != bytes(node)) + and checksum_address not in marked_nodes + for checksum_address, node in new_nodes.items()) + + remote_nodes_slashed = any(checksum_address in self._nodes for checksum_address in marked_nodes) + + if this_node_changed or remote_nodes_updated or remote_nodes_slashed: + # TODO: if nodes were kept in a Merkle tree, + # we'd have to only recalculate log(N) checksums. + # Is it worth it? + nodes = dict(self._nodes) + nodes.update(new_nodes) + for checksum_address in marked_nodes: + if checksum_address in nodes: + del nodes[checksum_address] + + all_nodes_sorted = sorted(itertools.chain(this_node_list, nodes.values()), + key=lambda node: node.checksum_address) + joined_metadata = b"".join(bytes(node) for node in all_nodes_sorted) + checksum = keccak_digest(joined_metadata).hex() + else: + nodes = self._nodes + checksum = self.checksum + + return FleetState(checksum=checksum, + nodes=nodes, + this_node_ref=self._this_node_ref, + this_node_metadata=this_node_metadata) + + @property + def population(self): + """Returns the number of all known nodes, including itself, if applicable.""" + return len(self) + int(self._this_node_metadata is not None) def __getitem__(self, checksum_address): return self._nodes[checksum_address] + def addresses(self): + return self._nodes.keys() + def __bool__(self): - return bool(self._nodes) + return len(self) != 0 def __contains__(self, item): - return item in self._nodes.keys() or item in self._nodes.values() + if isinstance(item, str): + return item in self._nodes + else: + return item.checksum_address in self._nodes def __iter__(self): yield from self._nodes.values() @@ -76,100 +170,182 @@ class FleetSensor: def __len__(self): return len(self._nodes) - def __eq__(self, other): - return self._nodes == other._nodes - - def __repr__(self): - return self._nodes.__repr__() - - def population(self): - return len(self) + len(self.additional_nodes_to_track) - - @property - def checksum(self): - return self._checksum - - @checksum.setter - def checksum(self, checksum_value): - self._checksum = checksum_value - self._nickname = Nickname.from_seed(checksum_value, length=1) - - @property - def nickname(self): - return self._nickname - - @property - def icon(self) -> str: - if self.nickname is NO_KNOWN_NODES: - return str(NO_KNOWN_NODES) - return self.nickname.icon - - def addresses(self): - return self._nodes.keys() - + # TODO: we only send it along with `FLEET_STATES_MATCH`, so it is essentially useless. + # But it's hard to change now because older nodes will be looking for it. def snapshot(self): - fleet_state_checksum_bytes = binascii.unhexlify(self.checksum) - fleet_state_updated_bytes = self.updated.epoch.to_bytes(4, byteorder="big") - return fleet_state_checksum_bytes + fleet_state_updated_bytes + checksum_bytes = binascii.unhexlify(self.checksum) + timestamp_bytes = self.timestamp.epoch.to_bytes(4, byteorder="big") + return checksum_bytes + timestamp_bytes - def record_fleet_state(self, additional_nodes_to_track=None): - if additional_nodes_to_track: - self.additional_nodes_to_track.extend(additional_nodes_to_track) + snapshot_splitter = BytestringSplitter(32, 4) - if not self._nodes: - # No news here. - return - sorted_nodes = self.sorted() - - sorted_nodes_joined = b"".join(bytes(n) for n in sorted_nodes) - checksum = keccak_digest(sorted_nodes_joined).hex() - if checksum not in self.states: - self.checksum = keccak_digest(b"".join(bytes(n) for n in self.sorted())).hex() - self.updated = maya.now() - # For now we store the sorted node list. Someday we probably spin this out into - # its own class, FleetState, and use it as the basis for partial updates. - new_state = self.FleetState(nickname=self.nickname, - nodes=sorted_nodes, - icon=self.icon, - updated=self.updated, - checksum=self.checksum) - self.states[checksum] = new_state - return checksum, new_state - - def start_tracking_state(self, additional_nodes_to_track=None): - if additional_nodes_to_track is None: - additional_nodes_to_track = list() - self.additional_nodes_to_track.extend(additional_nodes_to_track) - self._tracking = True - self.update_fleet_state() - - def sorted(self): - nodes_to_consider = list(self._nodes.values()) + self.additional_nodes_to_track - return sorted(nodes_to_consider, key=lambda n: n.checksum_address) + @staticmethod + def unpack_snapshot(data): + checksum_bytes, timestamp_bytes, remainder = FleetState.snapshot_splitter(data, return_remainder=True) + checksum = checksum_bytes.hex() + timestamp = maya.MayaDT(int.from_bytes(timestamp_bytes, byteorder="big")) + return checksum, timestamp, remainder def shuffled(self): nodes_we_know_about = list(self._nodes.values()) random.shuffle(nodes_we_know_about) return nodes_we_know_about - def abridged_states_dict(self): - abridged_states = {} - for k, v in self.states.items(): - abridged_states[k] = self.abridged_state_details(v) - return abridged_states + def to_json(self): + return dict(nickname=self.nickname.to_json(), + updated=self.timestamp.rfc2822()) + + @property + def icon(self) -> str: + # FIXME: should it be called at all if there are no states recorded? + if len(self) == 0: + return str(NO_KNOWN_NODES) + return self.nickname.icon + + def items(self): + return self._nodes.items() + + def values(self): + return self._nodes.values() + + def __repr__(self): + return f"FleetState({self.checksum}, {self._nodes}, {self._this_node_ref}, {self._this_node_metadata})" + + +class FleetSensor: + """ + A representation of a fleet of NuCypher nodes. + + If `this_node` is provided, it will be included in the state checksum + (but not returned during iteration/lookups). + """ + log = Logger("Learning") + + def __init__(self, domain: str, this_node=None): + + self._domain = domain + + self._current_state = FleetState.new(this_node) + self._archived_states = [self._current_state.archived()] + self.remote_states = {} + + # temporary accumulator for new nodes to avoid updating the fleet state every time + self._new_nodes = {} + self._marked = set() # Beginning of bucketing. + + self._auto_update_state = False + + def record_node(self, node): + + if node.domain == self._domain: + self._new_nodes[node.checksum_address] = node + + if self._auto_update_state: + self.log.info(f"Updating fleet state after saving node {node}") + self.record_fleet_state() + else: + msg = f"Rejected node {node} because its domain is '{node.domain}' but we're only tracking '{self._domain}'" + self.log.warn(msg) + + def __getitem__(self, item): + return self._current_state[item] + + def __bool__(self): + return bool(self._current_state) + + def __contains__(self, item): + """ + Checks if the node *with the same metadata* is recorded in the current state. + Does not compare ``item`` with the owner node of this FleetSensor. + """ + return item in self._current_state + + def __iter__(self): + yield from self._current_state + + def __len__(self): + return len(self._current_state) + + def __repr__(self): + return f"FleetSensor({self._current_state.__repr__()})" + + @property + def current_state(self): + return self._current_state + + @property + def checksum(self): + # FIXME: should it be called at all if there are no states recorded? + if self._current_state.population == 0: + return NO_KNOWN_NODES + return self._current_state.checksum + + @property + def population(self): + return self._current_state.population + + @property + def nickname(self): + # FIXME: should it be called at all if there are no states recorded? + if self._current_state.population == 0: + return NO_KNOWN_NODES + return self._current_state.nickname + + @property + def icon(self) -> str: + # FIXME: should it be called at all if there are no states recorded? + return self._current_state.icon + + @property + def timestamp(self): + return self._current_state.timestamp + + def items(self): + return self._current_state.items() + + def values(self): + return self._current_state.values() + + def latest_states(self, quantity): + """ + Returns at most ``quantity`` latest archived states (including the current one), + in chronological order. + """ + return self._archived_states[-min(len(self._archived_states), quantity):] + + def addresses(self): + return self._current_state.addresses() + + def snapshot(self): + return self._current_state.snapshot() @staticmethod - def abridged_state_details(state): - return {"nickname": str(state.nickname), - # FIXME: generalize in case we want to extend the number of symbols in the state nickname - "symbol": state.nickname.characters[0].symbol, - "color_hex": state.nickname.characters[0].color_hex, - "color_name": state.nickname.characters[0].color_name, - "updated": state.updated.rfc2822(), - } + def unpack_snapshot(data): + return FleetState.unpack_snapshot(data) + + def record_fleet_state(self): + new_state = self._current_state.with_updated_nodes(self._new_nodes, self._marked) + self._new_nodes = {} + self._marked = set() + self._current_state = new_state + + # TODO: set a limit on the number of archived states? + # Two ways to collect archived states: + # 1. (current) add a state to the archive every time it changes + # 2. (possible) keep a dictionary of known states + # and bump the timestamp of a previously encountered one + if new_state.checksum != self._archived_states[-1].checksum: + self._archived_states.append(new_state.archived()) + + def shuffled(self): + return self._current_state.shuffled() def mark_as(self, label: Exception, node: "Teacher"): - self._marked[label].append(node) + # TODO: for now we're not using `label` in any way, so we're just ignoring it + self._marked.add(node.checksum_address) - if self._nodes.get(node): - del self._nodes[node] + def record_remote_fleet_state(self, checksum_address, state_checksum, timestamp, population): + # TODO: really we can just create the timestamp here + + nickname = Nickname.from_seed(state_checksum, length=1) # TODO: create in a single place + self.remote_states[checksum_address] = ArchivedFleetState(state_checksum, nickname, timestamp, population) diff --git a/nucypher/characters/base.py b/nucypher/characters/base.py index d3cd2a8bf..8c98622ba 100644 --- a/nucypher/characters/base.py +++ b/nucypher/characters/base.py @@ -73,6 +73,7 @@ class Character(Learner): provider_uri: str = None, signer: Signer = None, registry: BaseContractRegistry = None, + include_self_in_the_state: bool = False, *args, **kwargs ) -> None: @@ -191,6 +192,7 @@ class Character(Learner): domain=domain, network_middleware=self.network_middleware, node_class=known_node_class, + include_self_in_the_state=include_self_in_the_state, *args, **kwargs) # diff --git a/nucypher/characters/lawful.py b/nucypher/characters/lawful.py index 633253744..d71bbcb01 100644 --- a/nucypher/characters/lawful.py +++ b/nucypher/characters/lawful.py @@ -981,7 +981,7 @@ class Bob(Character): # Not enough matching nodes. Fine, we'll just publish to the first few. try: # TODO: This is almost certainly happening in a test. If it does happen in production, it's a bit of a problem. Need to fix #2124 to mitigate. - target_nodes = list(nodes._nodes.values())[0:6] + target_nodes = list(nodes.values())[0:6] return target_nodes except IndexError: raise self.NotEnoughNodes("There aren't enough nodes on the network to enact this policy. Unless this is day one of the network and nodes are still getting spun up, something is bonkers.") @@ -1113,6 +1113,7 @@ class Ursula(Teacher, Character, Worker): known_nodes=known_nodes, domain=domain, known_node_class=Ursula, + include_self_in_the_state=True, **character_kwargs) if is_me: @@ -1232,7 +1233,7 @@ class Ursula(Teacher, Character, Worker): decentralized_identity_evidence=decentralized_identity_evidence) if is_me: - self.known_nodes.record_fleet_state(additional_nodes_to_track=[self]) # Initial Impression + self.known_nodes.record_fleet_state() # Initial Impression message = "THIS IS YOU: {}: {}".format(self.__class__.__name__, self) self.log.info(message) @@ -1241,6 +1242,11 @@ class Ursula(Teacher, Character, Worker): message = "Initialized Stranger {} | {}".format(self.__class__.__name__, self) self.log.debug(message) + # FIXME: we need to know when Ursula is ready to be recorded in a fleet state. + # The first time fleet state is updated may be still within this constructor. + # Any better way to solve this? + self.finished_initializing = True + def __prune_datastore(self) -> None: """Deletes all expired arrangements, kfrags, and treasure maps in the datastore.""" now = maya.MayaDT.from_datetime(datetime.fromtimestamp(self._datastore_pruning_task.clock.seconds())) diff --git a/nucypher/cli/painting/nodes.py b/nucypher/cli/painting/nodes.py index 67c67d701..5caf06c9e 100644 --- a/nucypher/cli/painting/nodes.py +++ b/nucypher/cli/painting/nodes.py @@ -15,7 +15,6 @@ You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ import maya -from constant_sorrow.constants import NO_KNOWN_NODES from nucypher.config.constants import SEEDNODES from nucypher.datastore.datastore import RecordNotFound @@ -23,19 +22,7 @@ from nucypher.datastore.models import Workorder def build_fleet_state_status(ursula) -> str: - # Build FleetState status line - if ursula.known_nodes.checksum is not NO_KNOWN_NODES: - fleet_state_checksum = ursula.known_nodes.checksum[:7] - fleet_state_nickname = ursula.known_nodes.nickname - fleet_state = '{checksum} ⇀{nickname}↽ {icon}'.format(icon=fleet_state_nickname.icon, - nickname=fleet_state_nickname, - checksum=fleet_state_checksum) - elif ursula.known_nodes.checksum is NO_KNOWN_NODES: - fleet_state = 'No Known Nodes' - else: - fleet_state = 'Unknown' - - return fleet_state + return str(ursula.known_nodes.current_state) def paint_node_status(emitter, ursula, start_time): @@ -61,7 +48,7 @@ def paint_node_status(emitter, ursula, start_time): except RecordNotFound: num_work_orders = 0 - stats = ['⇀URSULA {}↽'.format(ursula.nickname_icon), + stats = ['⇀URSULA {}↽'.format(ursula.nickname.icon), '{}'.format(ursula), 'Uptime .............. {}'.format(maya.now() - start_time), 'Start Time .......... {}'.format(start_time.slang_time()), diff --git a/nucypher/cli/processes.py b/nucypher/cli/processes.py index 4895221ee..3bdb018da 100644 --- a/nucypher/cli/processes.py +++ b/nucypher/cli/processes.py @@ -114,8 +114,7 @@ class UrsulaCommandProtocol(LineReceiver): Display information about the network-wide fleet state as the attached Ursula node sees it. """ from nucypher.cli.painting.nodes import build_fleet_state_status - line = '{}'.format(build_fleet_state_status(ursula=self.ursula)) - self.emitter.echo(line) + self.emitter.echo(build_fleet_state_status(ursula=self.ursula)) def connectionMade(self): self.emitter.echo("\nType 'help' or '?' for help") diff --git a/nucypher/network/nodes.py b/nucypher/network/nodes.py index dba2f24dc..dc26a410b 100644 --- a/nucypher/network/nodes.py +++ b/nucypher/network/nodes.py @@ -174,7 +174,6 @@ class Learner: tracker_class = FleetSensor invalid_metadata_message = "{} has invalid metadata. The node's stake may have ended, or it is transitioning to a new interface. Ignoring." - fleet_state_population = None _DEBUG_MODE = False @@ -209,6 +208,7 @@ class Learner: abort_on_learning_error: bool = False, lonely: bool = False, verify_node_bonding: bool = True, + include_self_in_the_state: bool = False, ) -> None: self.log = Logger("learning-loop") # type: Logger @@ -228,7 +228,7 @@ class Learner: self._learning_listeners = defaultdict(list) self._node_ids_to_learn_about_immediately = set() - self.__known_nodes = self.tracker_class(domain=domain) + self.__known_nodes = self.tracker_class(domain=domain, this_node=self if include_self_in_the_state else None) self._verify_node_bonding = verify_node_bonding self.lonely = lonely @@ -253,6 +253,7 @@ class Learner: self.remember_node(node, eager=True) except self.UnresponsiveTeacher: self.unresponsive_startup_nodes.append(node) + self.known_nodes.record_fleet_state() self.teacher_nodes = deque() self._current_teacher_node = None # type: Teacher @@ -387,7 +388,7 @@ class Learner: # This node is already known. We can safely return. return False - self.known_nodes[node.checksum_address] = node # FIXME - dont always remember nodes, bucket them. + self.known_nodes.record_node(node) # FIXME - dont always remember nodes, bucket them. if self.save_metadata: self.node_storage.store_node_metadata(node=node) @@ -749,7 +750,7 @@ class Learner: if not self.done_seeding: try: - remembered_seednodes = self.load_seednodes(record_fleet_state=False) + remembered_seednodes = self.load_seednodes(record_fleet_state=True) except Exception as e: # Even if we aren't aborting on learning errors, we want this to crash the process pronto. e.crash_right_now = True @@ -761,7 +762,7 @@ class Learner: current_teacher = self.current_teacher_node() # Will raise if there's no available teacher. - if Teacher in self.__class__.__bases__: + if isinstance(self, Teacher): announce_nodes = [self] else: announce_nodes = None @@ -842,18 +843,18 @@ class Learner: f"Invalid signature ({signature}) received from teacher {current_teacher} for payload {node_payload}") # End edge case handling. - payload = FleetSensor.snapshot_splitter(node_payload, return_remainder=True) - fleet_state_checksum_bytes, fleet_state_updated_bytes, node_payload = payload + + fleet_state_checksum, fleet_state_updated, node_payload = FleetSensor.unpack_snapshot(node_payload) current_teacher.last_seen = maya.now() - # TODO: This is weird - let's get a stranger FleetState going. NRN - checksum = fleet_state_checksum_bytes.hex() if constant_or_bytes(node_payload) is FLEET_STATES_MATCH: - current_teacher.update_snapshot(checksum=checksum, - updated=maya.MayaDT( - int.from_bytes(fleet_state_updated_bytes, byteorder="big")), - number_of_known_nodes=self.known_nodes.population()) + self.known_nodes.record_remote_fleet_state( + current_teacher.checksum_address, + fleet_state_checksum, + fleet_state_updated, + self.known_nodes.population) + return FLEET_STATES_MATCH # Note: There was previously a version check here, but that required iterating through node bytestrings twice, @@ -908,9 +909,11 @@ class Learner: self.log.warn(message) # Is cycling happening in the right order? - current_teacher.update_snapshot(checksum=checksum, - updated=maya.MayaDT(int.from_bytes(fleet_state_updated_bytes, byteorder="big")), - number_of_known_nodes=len(sprouts)) + self.known_nodes.record_remote_fleet_state( + current_teacher.checksum_address, + fleet_state_checksum, + fleet_state_updated, + len(sprouts)) ################### @@ -945,13 +948,8 @@ class Teacher: # self.domain = domain - self.fleet_state_checksum = None - self.fleet_state_updated = None self.last_seen = NEVER_SEEN("No Connection to Node") - self.fleet_state_population = UNKNOWN_FLEET_STATE - self.fleet_state_nickname = UNKNOWN_FLEET_STATE - # # Identity # @@ -1050,23 +1048,6 @@ class Teacher: payload += ursulas_as_bytes return payload - def update_snapshot(self, checksum, updated, number_of_known_nodes): - """ - TODO: We update the simple snapshot here, but of course if we're dealing - with an instance that is also a Learner, it has - its own notion of its FleetState, so we probably - need a reckoning of sorts here to manage that. In time. NRN - - :param checksum: - :param updated: - :param number_of_known_nodes: - :return: - """ - self.fleet_state_nickname = Nickname.from_seed(checksum, length=1) - self.fleet_state_checksum = checksum - self.fleet_state_updated = updated - self.fleet_state_population = number_of_known_nodes - # # Stamp # @@ -1310,75 +1291,66 @@ class Teacher: return self.timestamp.epoch.to_bytes(4, 'big') # - # Nicknames and Metadata + # Status metadata # - @property - def nickname_icon(self): - return self.nickname.icon - - def nickname_icon_details(self): - return dict( - node_class=self.__class__.__name__, - version=self.TEACHER_VERSION, - # FIXME: generalize in case we want to extend the number of symbols in the node nickname - first_color=self.nickname.characters[0].color_hex, - first_symbol=self.nickname.characters[0].symbol, - second_color=self.nickname.characters[1].color_hex, - second_symbol=self.nickname.characters[1].symbol, - address_first6=self.checksum_address[2:8] - ) - - def known_nodes_details(self, raise_invalid=True) -> dict: - abridged_nodes = {} - for checksum_address, node in self.known_nodes._nodes.items(): - try: - abridged_nodes[checksum_address] = self.node_details(node=node) - except self.StampNotSigned: - if raise_invalid: - raise - self.log.error(f"encountered unsigned stamp for node with checksum: {checksum_address}") - return abridged_nodes - - @staticmethod - def node_details(node): - """Stranger-Safe Details""" - node.mature() + def _status_info_base(self, raise_invalid=True): try: - last_seen = node.last_seen.iso8601() + worker_address = self.worker_address + # FIXME: how did such a node end up in `known_nodes`? + except self.StampNotSigned: + if raise_invalid: + raise + self.log.error(f"encountered unsigned stamp for node with checksum: {self.checksum_address}") + worker_address = 'UNSIGNED_STAMP' + + return dict( + nickname=self.nickname.to_json(), + staker_address=self.checksum_address, + worker_address=worker_address, + rest_url=self.rest_url(), + ) + + def _status_info_remote(self, known_nodes, raise_invalid=True): + info = self._status_info_base(raise_invalid=raise_invalid) + + try: + last_seen = self.last_seen.iso8601() except AttributeError: - last_seen = str(node.last_seen) # In case it's the constant NEVER_SEEN + last_seen = None # In case it's the constant NEVER_SEEN - fleet_icon = node.fleet_state_nickname - if fleet_icon is UNKNOWN_FLEET_STATE: - fleet_icon = "?" # TODO NRN, MN + info['last_learned_from'] = last_seen + + # TODO: what *is* the `timestamp`, anyway? When is it created? + info['timestamp'] = self.timestamp.iso8601() + + # TODO: how come we know about the node but not about its fleet state? Does it ever happen? + if self.checksum_address in known_nodes.remote_states: + info['recorded_fleet_state'] = known_nodes.remote_states[self.checksum_address].to_json() else: - fleet_icon = fleet_icon.icon + info['recorded_fleet_state'] = None - payload = {"icon_details": node.nickname.payload(), - "rest_url": node.rest_url(), - "nickname": str(node.nickname), - "worker_address": node.worker_address, - "staker_address": node.checksum_address, - "timestamp": node.timestamp.iso8601(), - "last_seen": last_seen, - "fleet_state": node.fleet_state_checksum or 'unknown', - "fleet_state_icon": fleet_icon, - "domain": node.domain, - 'version': nucypher.__version__ - } - return payload + return info - def abridged_node_details(self, raise_invalid=True) -> dict: - """Self-Reporting""" - payload = self.node_details(node=self) - states = self.known_nodes.abridged_states_dict() - known = self.known_nodes_details(raise_invalid=raise_invalid) - payload.update({'states': states, 'known_nodes': known}) - if not self.federated_only: - payload.update({ - "balances": dict(eth=float(self.eth_balance), nu=float(self.token_balance.to_tokens())), - "missing_commitments": self.missing_commitments, - "last_committed_period": self.last_committed_period}) - return payload + def status_info(self, raise_invalid=True, omit_known_nodes=False): + # FIXME: is anyone using `raise_invalid=True`? Or is it always `False`? + + for node in self.known_nodes: + node.mature() + + info = self._status_info_base() + + info['domain'] = self.domain + info['version'] = nucypher.__version__ + + latest_states = self.known_nodes.latest_states(5) + + info['fleet_state'] = latest_states[-1].to_json() + info['previous_fleet_states'] = [state.to_json() for state in latest_states[:-1]] + + if not omit_known_nodes: + info['known_nodes'] = [node._status_info_remote(self.known_nodes, raise_invalid=raise_invalid) + for node in self.known_nodes.values()] + + return info diff --git a/nucypher/network/server.py b/nucypher/network/server.py index f1037cfa3..433e3f0a0 100644 --- a/nucypher/network/server.py +++ b/nucypher/network/server.py @@ -173,7 +173,7 @@ def _make_rest_app(datastore: Datastore, this_node, domain: str, log: Logger) -> # TODO: Is this every something we don't want to do? this_node.start_learning_loop() - if this_node.known_nodes.checksum is NO_KNOWN_NODES: + if not this_node.known_nodes: return Response(b"", headers=headers, status=204) known_nodes_bytestring = this_node.bytestring_of_known_nodes() @@ -422,30 +422,28 @@ def _make_rest_app(datastore: Datastore, this_node, domain: str, log: Logger) -> @rest_app.route('/status/', methods=['GET']) def status(): - if request.args.get('json'): - payload = this_node.abridged_node_details(raise_invalid=False) - response = jsonify(payload) - return response + + return_json = request.args.get('json') == 'true' + omit_known_nodes = request.args.get('omit_known_nodes') == 'true' + + status_info = this_node.status_info(raise_invalid=False, + omit_known_nodes=omit_known_nodes) + + if return_json: + return jsonify(status_info) else: headers = {"Content-Type": "text/html", "charset": "utf-8"} - previous_states = list(reversed(this_node.known_nodes.states.values()))[:5] - # Mature every known node before rendering. - for node in this_node.known_nodes: - node.mature() try: - content = status_template.render(this_node=this_node, - known_nodes=this_node.known_nodes, - previous_states=previous_states, - domain=domain, - version=nucypher.__version__, - checksum_address=this_node.checksum_address) + content = status_template.render(status_info) + except Exception as e: text_error = mako_exceptions.text_error_template().render() html_error = mako_exceptions.html_error_template().render() log.debug("Template Rendering Exception:\n" + text_error) return Response(response=html_error, headers=headers, status=500) + return Response(response=content, headers=headers) return rest_app diff --git a/nucypher/network/templates/basic_status.mako b/nucypher/network/templates/basic_status.mako index 3e1e17c6f..72e80e129 100644 --- a/nucypher/network/templates/basic_status.mako +++ b/nucypher/network/templates/basic_status.mako @@ -6,9 +6,6 @@ def hex_to_rgb(color_hex): b = int(color_hex[5:7], 16) return r, g, b -def rgb_to_hex(r, g, b): - return f"#{r:02x}{g:02x}{b:02x}" - def contrast_color(color_hex): r, g, b = hex_to_rgb(color_hex) # As defined in https://www.w3.org/WAI/ER/WD-AERT/#color-contrast @@ -20,23 +17,25 @@ def contrast_color(color_hex): return "white" def character_span(character): - return f'{character.symbol}' + color = character['color_hex'] + symbol = character['symbol'] + return f'{symbol}' %> -<%def name="fleet_state_icon(checksum, nickname, population)"> -%if not checksum: -NO FLEET STATE AVAILABLE +<%def name="fleet_state_icon(state)"> +%if not state: +— %else: - +
## Need to compose these spans as strings to avoid introducing whitespaces - ${"".join(character_span(character) for character in nickname.characters)} + ${"".join(character_span(character) for character in state['nickname']['characters'])} - ${population} nodes + ${state['population']} nodes
- ${checksum[0:8]} + ${state['checksum'][0:8]}
@@ -44,30 +43,20 @@ NO FLEET STATE AVAILABLE -<%def name="fleet_state_icon_from_state(state)"> -${fleet_state_icon(state.checksum, state.nickname, len(state.nodes))} - - - -<%def name="fleet_state_icon_from_known_nodes(state)"> -${fleet_state_icon(state.checksum, state.nickname, state.population())} - - - <%def name="node_info(node)">
## Need to compose these spans as strings to avoid introducing whitespaces - ${"".join(character_span(character) for character in node.nickname.characters)} + ${"".join(character_span(character) for character in node['nickname']['characters'])} - - ${ node.nickname } + + ${ node['nickname']['text'] }
- ${ node.checksum_address } + ${ node['staker_address'] }
@@ -75,7 +64,7 @@ ${fleet_state_icon(state.checksum, state.nickname, state.population())} -<%def name="main()"> +<%def name="main(status_info)"> @@ -168,7 +157,7 @@ ${fleet_state_icon(state.checksum, state.nickname, state.population())} - + @@ -176,48 +165,54 @@ ${fleet_state_icon(state.checksum, state.nickname, state.population())} - + - + - +
${node_info(this_node)}
${node_info(status_info)}
Running:v${ version }v${ status_info['version'] }
Domain:${ domain }${ status_info['domain'] }
Fleet state:${fleet_state_icon_from_known_nodes(this_node.known_nodes)}${fleet_state_icon(status_info['fleet_state'])}
Previous states: - %for state in previous_states: - ${fleet_state_icon_from_state(state)} + %for state in status_info['previous_fleet_states']: + ${fleet_state_icon(state)} %endfor
-

${len(known_nodes)} ${"known node" if len(known_nodes) == 1 else "known nodes"}:

+ %if 'known_nodes' in status_info: +

${len(status_info['known_nodes'])} ${"known node" if len(status_info['known_nodes']) == 1 else "known nodes"}:

- + - %for node in known_nodes: + %for node in status_info['known_nodes']: - - - + + + %endfor
LaunchedLast SeenLast Learned From Fleet State
${node_info(node)}${ node.timestamp }${ node.last_seen }${fleet_state_icon(node.fleet_state_checksum, - node.fleet_state_nickname, - node.fleet_state_population)}${node['timestamp']} + %if node['last_learned_from']: + ${node['last_learned_from']} + %else: + — + %endif + ${fleet_state_icon(node['recorded_fleet_state'])}
+ %endif diff --git a/nucypher/network/trackers.py b/nucypher/network/trackers.py index bb18f3b9f..0218f16af 100644 --- a/nucypher/network/trackers.py +++ b/nucypher/network/trackers.py @@ -188,7 +188,7 @@ class AvailabilityTracker: return def sample(self, quantity: int) -> list: - population = tuple(self._ursula.known_nodes._nodes.values()) + population = tuple(self._ursula.known_nodes.values()) ursulas = random.sample(population=population, k=quantity) return ursulas diff --git a/nucypher/utilities/prometheus/collector.py b/nucypher/utilities/prometheus/collector.py index 07e287a20..df09f9951 100644 --- a/nucypher/utilities/prometheus/collector.py +++ b/nucypher/utilities/prometheus/collector.py @@ -118,7 +118,7 @@ class UrsulaInfoMetricsCollector(BaseMetricsCollector): 'host': str(self.ursula.rest_interface), 'domain': self.ursula.domain, 'nickname': str(self.ursula.nickname), - 'nickname_icon': self.ursula.nickname_icon, + 'nickname_icon': self.ursula.nickname.icon, 'fleet_state': str(self.ursula.known_nodes.checksum), 'known_nodes': str(len(self.ursula.known_nodes)) } diff --git a/tests/acceptance/characters/test_ursula_web_status.py b/tests/acceptance/characters/test_ursula_web_status.py index 9bd014337..22b9b7b7f 100644 --- a/tests/acceptance/characters/test_ursula_web_status.py +++ b/tests/acceptance/characters/test_ursula_web_status.py @@ -46,9 +46,11 @@ def test_ursula_html_renders(ursula, client): assert str(ursula.nickname).encode() in response.data -def test_decentralized_json_status_endpoint(ursula, client): - response = client.get('/status/?json=true') +@pytest.mark.parametrize('omit_known_nodes', [False, True]) +def test_decentralized_json_status_endpoint(ursula, client, omit_known_nodes): + omit_known_nodes_str = 'true' if omit_known_nodes else 'false' + response = client.get(f'/status/?json=true&omit_known_nodes={omit_known_nodes_str}') assert response.status_code == 200 json_status = response.get_json() - status = ursula.abridged_node_details() + status = ursula.status_info(omit_known_nodes=omit_known_nodes) assert json_status == status diff --git a/tests/acceptance/network/test_network_actors.py b/tests/acceptance/network/test_network_actors.py index a72384cb5..a048d5760 100644 --- a/tests/acceptance/network/test_network_actors.py +++ b/tests/acceptance/network/test_network_actors.py @@ -118,9 +118,6 @@ def test_vladimir_illegal_interface_key_does_not_propagate(blockchain_ursulas): # Indeed, Ursula noticed that something was up. vladimir in other_ursula.suspicious_activities_witnessed['vladimirs'] - # She marked him as Invalid... - vladimir in other_ursula.known_nodes._marked[vladimir.InvalidNode] - # ...and booted him from known_nodes vladimir not in other_ursula.known_nodes @@ -143,7 +140,8 @@ def test_alice_refuses_to_make_arrangement_unless_ursula_is_valid(blockchain_ali # Ideally, a fishy node shouldn't be present in `known_nodes`, # but I guess we're testing the case when it became fishy somewhere between we learned about it # and the proposal arrangement. - blockchain_alice.known_nodes[vladimir.checksum_address] = vladimir + blockchain_alice.known_nodes.record_node(vladimir) + blockchain_alice.known_nodes.record_fleet_state() with pytest.raises(vladimir.InvalidNode): idle_blockchain_policy._propose_arrangement(address=vladimir.checksum_address, diff --git a/tests/fixtures.py b/tests/fixtures.py index 54abe2099..c564486ea 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -954,8 +954,8 @@ def fleet_of_highperf_mocked_ursulas(ursula_federated_test_config, request): all_ursulas = {u.checksum_address: u for u in _ursulas} for ursula in _ursulas: - ursula.known_nodes._nodes = all_ursulas - ursula.known_nodes.checksum = b"This is a fleet state checksum..".hex() + ursula.known_nodes.current_state._nodes = all_ursulas + ursula.known_nodes.current_state.checksum = b"This is a fleet state checksum..".hex() yield _ursulas for ursula in _ursulas: @@ -972,7 +972,7 @@ def highperf_mocked_alice(fleet_of_highperf_mocked_ursulas): save_metadata=False, reload_metadata=False) - with mock_cert_storage, mock_verify_node, mock_record_fleet_state, mock_message_verification, mock_keep_learning: + with mock_cert_storage, mock_verify_node, mock_message_verification, mock_keep_learning: alice = config.produce(known_nodes=list(fleet_of_highperf_mocked_ursulas)[:1]) yield alice # TODO: Where does this really, truly belong? diff --git a/tests/integration/characters/test_ursula_startup.py b/tests/integration/characters/test_ursula_startup.py index 07dbce067..adae74a99 100644 --- a/tests/integration/characters/test_ursula_startup.py +++ b/tests/integration/characters/test_ursula_startup.py @@ -24,7 +24,8 @@ def test_new_federated_ursula_announces_herself(lonely_ursula_maker): ursula_in_a_house, ursula_with_a_mouse = lonely_ursula_maker(quantity=2, domain="useless_domain") # Neither Ursula knows about the other. - assert ursula_in_a_house.known_nodes == ursula_with_a_mouse.known_nodes + assert ursula_with_a_mouse not in ursula_in_a_house.known_nodes + assert ursula_in_a_house not in ursula_with_a_mouse.known_nodes ursula_in_a_house.remember_node(ursula_with_a_mouse) diff --git a/tests/integration/learning/test_domains.py b/tests/integration/learning/test_domains.py index e61a94dfc..2fc0d071f 100644 --- a/tests/integration/learning/test_domains.py +++ b/tests/integration/learning/test_domains.py @@ -114,7 +114,7 @@ def test_learner_ignores_stored_nodes_from_other_domains(lonely_ursula_maker, tm # Once pest made its way into learner, learner taught passed it to other mainnet nodes. - learner.known_nodes._nodes[pest.checksum_address] = pest # This used to happen anyway. + learner.known_nodes.record_node(pest) # This used to happen anyway. other_staker._current_teacher_node = learner other_staker.learn_from_teacher_node() # And once it did, the node from the wrong domain spread. assert pest not in other_staker.known_nodes # But not anymore. diff --git a/tests/integration/learning/test_firstula_circumstances.py b/tests/integration/learning/test_firstula_circumstances.py index f729af8f4..f14f91635 100644 --- a/tests/integration/learning/test_firstula_circumstances.py +++ b/tests/integration/learning/test_firstula_circumstances.py @@ -51,7 +51,7 @@ def test_get_cert_from_running_seed_node(lonely_ursula_maker): network_middleware=RestMiddleware()).pop() assert not any_other_ursula.known_nodes - yield deferToThread(any_other_ursula.load_seednodes) + yield deferToThread(lambda: any_other_ursula.load_seednodes(record_fleet_state=True)) assert firstula in any_other_ursula.known_nodes firstula_as_learned = any_other_ursula.known_nodes[firstula.checksum_address] diff --git a/tests/integration/learning/test_fleet_state.py b/tests/integration/learning/test_fleet_state.py index 97c5285b9..e8b33d552 100644 --- a/tests/integration/learning/test_fleet_state.py +++ b/tests/integration/learning/test_fleet_state.py @@ -65,19 +65,30 @@ def test_old_state_is_preserved(federated_ursulas, lonely_ursula_maker): some_ursula_in_the_fleet = list(federated_ursulas)[0] lonely_learner.remember_node(some_ursula_in_the_fleet) checksum_after_learning_one = lonely_learner.known_nodes.checksum + assert some_ursula_in_the_fleet in lonely_learner.known_nodes + assert some_ursula_in_the_fleet.checksum_address in lonely_learner.known_nodes + assert len(lonely_learner.known_nodes) == 1 + assert lonely_learner.known_nodes.population == 2 another_ursula_in_the_fleet = list(federated_ursulas)[1] lonely_learner.remember_node(another_ursula_in_the_fleet) checksum_after_learning_two = lonely_learner.known_nodes.checksum + assert some_ursula_in_the_fleet in lonely_learner.known_nodes + assert another_ursula_in_the_fleet in lonely_learner.known_nodes + assert some_ursula_in_the_fleet.checksum_address in lonely_learner.known_nodes + assert another_ursula_in_the_fleet.checksum_address in lonely_learner.known_nodes + assert len(lonely_learner.known_nodes) == 2 + assert lonely_learner.known_nodes.population == 3 assert checksum_after_learning_one != checksum_after_learning_two - proper_first_state = sorted([some_ursula_in_the_fleet, lonely_learner], key=lambda n: n.checksum_address) - assert lonely_learner.known_nodes.states[checksum_after_learning_one].nodes == proper_first_state + first_state = lonely_learner.known_nodes._archived_states[-2] + assert first_state.population == 2 + assert first_state.checksum == checksum_after_learning_one - proper_second_state = sorted([some_ursula_in_the_fleet, another_ursula_in_the_fleet, lonely_learner], - key=lambda n: n.checksum_address) - assert lonely_learner.known_nodes.states[checksum_after_learning_two].nodes == proper_second_state + second_state = lonely_learner.known_nodes._archived_states[-1] + assert second_state.population == 3 + assert second_state.checksum == checksum_after_learning_two def test_state_is_recorded_after_learning(federated_ursulas, lonely_ursula_maker): @@ -87,22 +98,27 @@ def test_state_is_recorded_after_learning(federated_ursulas, lonely_ursula_maker """ _lonely_ursula_maker = partial(lonely_ursula_maker, quantity=1) lonely_learner = _lonely_ursula_maker().pop() + states = lonely_learner.known_nodes._archived_states # This Ursula doesn't know about any nodes. assert len(lonely_learner.known_nodes) == 0 some_ursula_in_the_fleet = list(federated_ursulas)[0] lonely_learner.remember_node(some_ursula_in_the_fleet) - assert len(lonely_learner.known_nodes.states) == 1 # Saved a fleet state when we remembered this node. + assert len(states) == 2 # Saved a fleet state when we remembered this node. + + # The first fleet state is just us and the one about whom we learned, which is part of the fleet. + assert states[-1].population == 2 # The rest of the fucking owl. lonely_learner.learn_from_teacher_node() - states = list(lonely_learner.known_nodes.states.values()) - assert len(states) == 2 + # There are two new states: one created after seednodes are loaded, to select a teacher, + # and the second after we get the rest of the nodes from the seednodes. + assert len(states) == 4 - assert len(states[0].nodes) == 2 # The first fleet state is just us and the one about whom we learned, which is part of the fleet. - assert len(states[1].nodes) == len(federated_ursulas) + 1 # When we ran learn_from_teacher_node, we also loaded the rest of the fleet. + # When we ran learn_from_teacher_node, we also loaded the rest of the fleet. + assert states[-1].population == len(federated_ursulas) + 1 def test_teacher_records_new_fleet_state_upon_hearing_about_new_node(federated_ursulas, lonely_ursula_maker): @@ -110,18 +126,23 @@ def test_teacher_records_new_fleet_state_upon_hearing_about_new_node(federated_u lonely_learner = _lonely_ursula_maker().pop() some_ursula_in_the_fleet = list(federated_ursulas)[0] + lonely_learner.remember_node(some_ursula_in_the_fleet) + states = some_ursula_in_the_fleet.known_nodes._archived_states - teacher_states_before = list(some_ursula_in_the_fleet.known_nodes.states.values()) + states_before = len(states) lonely_learner.learn_from_teacher_node() - teacher_states_after = list(some_ursula_in_the_fleet.known_nodes.states.values()) + states_after = len(states) - # We added one fleet state. - len(teacher_states_after) == len(teacher_states_before) + 1 + # FIXME: some kind of a timeout is required here to wait for the learning to end + return + + # `some_ursula_in_the_fleet` learned about `lonely_learner` + assert states_before + 1 == states_after # The current fleet state of the Teacher... teacher_fleet_state_checksum = some_ursula_in_the_fleet.fleet_state_checksum # ...is the same as the learner, because both have learned about everybody at this point. - teacher_fleet_state_checksum in lonely_learner.known_nodes.states + assert teacher_fleet_state_checksum == states[-1].checksum diff --git a/tests/integration/network/test_failure_modes.py b/tests/integration/network/test_failure_modes.py index fb6a870ab..d065942d7 100644 --- a/tests/integration/network/test_failure_modes.py +++ b/tests/integration/network/test_failure_modes.py @@ -35,7 +35,7 @@ def test_alice_can_grant_even_when_the_first_nodes_she_tries_are_down(federated_ m, n = 2, 3 policy_end_datetime = maya.now() + datetime.timedelta(days=5) label = b"this_is_the_path_to_which_access_is_being_granted" - federated_alice.known_nodes._nodes = {} + federated_alice.known_nodes.current_state._nodes = {} federated_alice.network_middleware = NodeIsDownMiddleware() @@ -92,7 +92,7 @@ def test_alice_can_grant_even_when_the_first_nodes_she_tries_are_down(federated_ def test_node_has_changed_cert(federated_alice, federated_ursulas): - federated_alice.known_nodes._nodes = {} + federated_alice.known_nodes.current_state._nodes = {} federated_alice.network_middleware = NodeIsDownMiddleware() federated_alice.network_middleware.client.certs_are_broken = True diff --git a/tests/mock/performance_mocks.py b/tests/mock/performance_mocks.py index b96d7d7c4..7ecccc7c1 100644 --- a/tests/mock/performance_mocks.py +++ b/tests/mock/performance_mocks.py @@ -159,8 +159,7 @@ def do_not_create_cert(*args, **kwargs): def simple_remember(ursula, node, *args, **kwargs): - address = node.checksum_address - ursula.known_nodes[address] = node + ursula.known_nodes.record_node(node) class NotARestApp: diff --git a/tests/unit/test_external_ip_utilities.py b/tests/unit/test_external_ip_utilities.py index bb4268b16..bb724db44 100644 --- a/tests/unit/test_external_ip_utilities.py +++ b/tests/unit/test_external_ip_utilities.py @@ -32,8 +32,15 @@ from nucypher.utilities.networking import ( from tests.constants import MOCK_IP_ADDRESS +MOCK_NETWORK = 'holodeck' + + class Dummy: # Teacher - certificate_filepath = None + + def __init__(self, checksum_address): + self.checksum_address = checksum_address + self.certificate_filepath = None + self.domain = MOCK_NETWORK class GoodResponse: status_code = 200 @@ -45,7 +52,7 @@ class Dummy: # Teacher content = 'DUMMY 404' def mature(self): - return Dummy() + return self def verify_node(self, *args, **kwargs): pass @@ -53,6 +60,9 @@ class Dummy: # Teacher def rest_url(self): return MOCK_IP_ADDRESS + def __bytes__(self): + return self.checksum_address.encode() + @pytest.fixture(autouse=True) def mock_requests(mocker): @@ -66,14 +76,9 @@ def mock_client(mocker): yield mocker.patch.object(NucypherMiddlewareClient, 'invoke_method', return_value=Dummy.GoodResponse) -@pytest.fixture() -def mock_network(): - return 'holodeck' - - @pytest.fixture(autouse=True) -def mock_default_teachers(mocker, mock_network): - teachers = {mock_network: (MOCK_IP_ADDRESS, )} +def mock_default_teachers(mocker): + teachers = {MOCK_NETWORK: (MOCK_IP_ADDRESS, )} mocker.patch.dict(RestMiddleware.TEACHER_NODES, teachers) @@ -82,31 +87,33 @@ def test_get_external_ip_from_centralized_source(mock_requests): mock_requests.assert_called_once_with(url=CENTRALIZED_IP_ORACLE_URL) -def test_get_external_ip_from_empty_known_nodes(mock_requests, mock_network): - sensor = FleetSensor(domain=mock_network) +def test_get_external_ip_from_empty_known_nodes(mock_requests): + sensor = FleetSensor(domain=MOCK_NETWORK) assert len(sensor) == 0 get_external_ip_from_known_nodes(known_nodes=sensor) # skipped because there are no known nodes mock_requests.assert_not_called() -def test_get_external_ip_from_known_nodes_with_one_known_node(mock_requests, mock_network): - sensor = FleetSensor(domain=mock_network) - sensor._nodes['0xdeadbeef'] = Dummy() +def test_get_external_ip_from_known_nodes_with_one_known_node(mock_requests): + sensor = FleetSensor(domain=MOCK_NETWORK) + sensor.record_node(Dummy('0xdeadbeef')) + sensor.record_fleet_state() assert len(sensor) == 1 get_external_ip_from_known_nodes(known_nodes=sensor) # skipped because there are too few known nodes mock_requests.assert_not_called() -def test_get_external_ip_from_known_nodes(mock_client, mock_network): +def test_get_external_ip_from_known_nodes(mock_client): # Setup FleetSensor - sensor = FleetSensor(domain=mock_network) + sensor = FleetSensor(domain=MOCK_NETWORK) sample_size = 3 - sensor._nodes['0xdeadbeef'] = Dummy() - sensor._nodes['0xdeadllama'] = Dummy() - sensor._nodes['0xdeadmouse'] = Dummy() + sensor.record_node(Dummy('0xdeadbeef')) + sensor.record_node(Dummy('0xdeadllama')) + sensor.record_node(Dummy('0xdeadmouse')) + sensor.record_fleet_state() assert len(sensor) == sample_size # First sampled node replies @@ -120,19 +127,20 @@ def test_get_external_ip_from_known_nodes(mock_client, mock_network): assert mock_client.call_count == sample_size -def test_get_external_ip_from_known_nodes_client(mocker, mock_client, mock_network): +def test_get_external_ip_from_known_nodes_client(mocker, mock_client): # Setup FleetSensor - sensor = FleetSensor(domain=mock_network) + sensor = FleetSensor(domain=MOCK_NETWORK) sample_size = 3 - sensor._nodes['0xdeadbeef'] = Dummy() - sensor._nodes['0xdeadllama'] = Dummy() - sensor._nodes['0xdeadmouse'] = Dummy() + sensor.record_node(Dummy('0xdeadbeef')) + sensor.record_node(Dummy('0xdeadllama')) + sensor.record_node(Dummy('0xdeadmouse')) + sensor.record_fleet_state() assert len(sensor) == sample_size # Setup HTTP Client - mocker.patch.object(Ursula, 'from_teacher_uri', return_value=Dummy()) - teacher_uri = RestMiddleware.TEACHER_NODES[mock_network][0] + mocker.patch.object(Ursula, 'from_teacher_uri', return_value=Dummy('0xdeadpork')) + teacher_uri = RestMiddleware.TEACHER_NODES[MOCK_NETWORK][0] get_external_ip_from_known_nodes(known_nodes=sensor, sample_size=sample_size) assert mock_client.call_count == 1 # first node responded @@ -142,22 +150,22 @@ def test_get_external_ip_from_known_nodes_client(mocker, mock_client, mock_netwo assert endpoint == f'https://{teacher_uri}/ping' -def test_get_external_ip_default_teacher_unreachable(mocker, mock_network): +def test_get_external_ip_default_teacher_unreachable(mocker): for error in NodeSeemsToBeDown: # Default seednode is down mocker.patch.object(Ursula, 'from_teacher_uri', side_effect=error) - ip = get_external_ip_from_default_teacher(network=mock_network) + ip = get_external_ip_from_default_teacher(network=MOCK_NETWORK) assert ip is None -def test_get_external_ip_from_default_teacher(mocker, mock_client, mock_requests, mock_network): +def test_get_external_ip_from_default_teacher(mocker, mock_client, mock_requests): mock_client.return_value = Dummy.GoodResponse - teacher_uri = RestMiddleware.TEACHER_NODES[mock_network][0] - mocker.patch.object(Ursula, 'from_teacher_uri', return_value=Dummy()) + teacher_uri = RestMiddleware.TEACHER_NODES[MOCK_NETWORK][0] + mocker.patch.object(Ursula, 'from_teacher_uri', return_value=Dummy('0xdeadbeef')) # "Success" - ip = get_external_ip_from_default_teacher(network=mock_network) + ip = get_external_ip_from_default_teacher(network=MOCK_NETWORK) assert ip == MOCK_IP_ADDRESS # Check that the correct endpoint and function is targeted @@ -181,16 +189,17 @@ def test_get_external_ip_default_unknown_network(): determine_external_ip_address(known_nodes=sensor, network=unknown_domain) -def test_get_external_ip_cascade_failure(mocker, mock_network, mock_requests): +def test_get_external_ip_cascade_failure(mocker, mock_requests): first = mocker.patch('nucypher.utilities.networking.get_external_ip_from_known_nodes', return_value=None) second = mocker.patch('nucypher.utilities.networking.get_external_ip_from_default_teacher', return_value=None) third = mocker.patch('nucypher.utilities.networking.get_external_ip_from_centralized_source', return_value=None) - sensor = FleetSensor(domain=mock_network) - sensor._nodes['0xdeadbeef'] = Dummy() + sensor = FleetSensor(domain=MOCK_NETWORK) + sensor.record_node(Dummy('0xdeadbeef')) + sensor.record_fleet_state() with pytest.raises(UnknownIPAddress, match='External IP address detection failed'): - determine_external_ip_address(network=mock_network, known_nodes=sensor) + determine_external_ip_address(network=MOCK_NETWORK, known_nodes=sensor) first.assert_called_once() second.assert_called_once()