Nucypher
/
nucypher
mirror of https://github.com/nucypher/nucypher.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #2423 from jMyles/domains 

Moving domain check into FleetSensor.
pull/2402/head
Justin Holmes 2020-11-04 14:35:36 -07:00 committed by GitHub
parent 2eb6d1b986 ef29bec8e7
commit 579dc0819f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 131 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
newsfragments/2423.bugfix.rst Normal file
View File

@ -0,0 +1,3 @@
Domain "leakage", or nodes saving metadata about nodes from other domains (but never being able to verify them) was still possible because domain-checking only occurred in the high-level APIs (and not, for example, when checking metadata POSTed to the node_metadata_exchange endpoint). This fixes that (fixes #2417).
Additionally, domains are no longer separated into "serving" or "learning". Each Learner instance now has exactly one domain, and it is called domain.

21
nucypher/acumen/perception.py
View File

@ -42,22 +42,27 @@ class FleetSensor:
log = Logger("Learning")
FleetState = namedtuple("FleetState", ("nickname", "icon", "nodes", "updated", "checksum"))
def __init__(self):
def __init__(self, domain: str):
self.domain = domain
self.additional_nodes_to_track = []
self.updated = maya.now()
self._nodes = OrderedDict()
self._marked = defaultdict(list) # Beginning of bucketing.
self.states = OrderedDict()
def __setitem__(self, key, value):
self._nodes[key] = value
def __setitem__(self, checksum_address, node_or_sprout):
if node_or_sprout.domain == self.domain:
self._nodes[checksum_address] = node_or_sprout
if self._tracking:
self.log.info("Updating fleet state after saving node {}".format(value))
self.record_fleet_state()
if self._tracking:
self.log.info("Updating fleet state after saving node {}".format(node_or_sprout))
self.record_fleet_state()
else:
msg = f"Rejected node {node_or_sprout} because its domain is '{node_or_sprout.domain}' but we're only tracking '{self.domain}'"
self.log.warn(msg)
def __getitem__(self, item):
return self._nodes[item]
def __getitem__(self, checksum_address):
return self._nodes[checksum_address]
def __bool__(self):
return bool(self._nodes)

6
nucypher/characters/lawful.py
View File

@ -1111,7 +1111,7 @@ class Ursula(Teacher, Character, Worker):
rest_app, datastore = make_rest_app(
this_node=self,
db_filepath=db_filepath,
serving_domain=domain,
domain=domain,
)
# TLSHostingPower (Ephemeral Powers and Private Keys)
@ -1237,7 +1237,7 @@ class Ursula(Teacher, Character, Worker):
# if learning: # TODO: Include learning startup here with the rest of the services?
# self.start_learning_loop(now=self._start_learning_now)
# if emitter:
# emitter.message(f"✓ Node Discovery ({','.join(self.learning_domain)})", color='green')
# emitter.message(f"✓ Node Discovery ({','.join(self.domain)})", color='green')
if self._availability_check and availability:
self._availability_tracker.start(now=False) # wait...
@ -1352,7 +1352,7 @@ class Ursula(Teacher, Character, Worker):
as_bytes = bytes().join((version,
self.canonical_public_address,
bytes(VariableLengthBytestring(self.serving_domain.encode('utf-8'))),
bytes(VariableLengthBytestring(self.domain.encode('utf-8'))),
self.timestamp_bytes(),
bytes(self._interface_signature),
bytes(VariableLengthBytestring(self.decentralized_identity_evidence)), # FIXME: Fixed length doesn't work with federated

33
nucypher/network/nodes.py
View File

@ -102,6 +102,11 @@ class NodeSprout(PartiallyKwargifiedBytes):
def stamp(self) -> bytes:
return self.processed_objects['verifying_key'][0]
@property
def domain(self) -> str:
domain_bytes = PartiallyKwargifiedBytes.__getattr__(self, "domain")
return domain_bytes.decode("utf-8")
@property
def checksum_address(self):
if not self._checksum_address:
@ -210,7 +215,7 @@ class Learner:
self.log = Logger("learning-loop") # type: Logger
self.learning_deferred = Deferred()
self.learning_domain = domain
self.domain = domain
if not self.federated_only:
default_middleware = self.__DEFAULT_MIDDLEWARE_CLASS(registry=self.registry)
else:
@ -224,7 +229,7 @@ class Learner:
self._learning_listeners = defaultdict(list)
self._node_ids_to_learn_about_immediately = set()
self.__known_nodes = self.tracker_class()
self.__known_nodes = self.tracker_class(domain=domain)
self._verify_node_bonding = verify_node_bonding
self.lonely = lonely
@ -293,8 +298,8 @@ class Learner:
discovered = []
if self.learning_domain:
canonical_sage_uris = self.network_middleware.TEACHER_NODES.get(self.learning_domain, ())
if self.domain:
canonical_sage_uris = self.network_middleware.TEACHER_NODES.get(self.domain, ())
for uri in canonical_sage_uris:
try:
@ -350,18 +355,14 @@ class Learner:
restored_from_disk = []
invalid_nodes = defaultdict(list)
for node in stored_nodes:
try: # Workaround until #2356 is fixed
node_domain = node.domain.decode('utf-8')
except:
node_domain = node.serving_domain
if node_domain != self.learning_domain:
invalid_nodes[node_domain].append(node)
if node.domain != self.domain:
invalid_nodes[node.domain].append(node)
continue
restored_node = self.remember_node(node, record_fleet_state=False) # TODO: Validity status 1866
restored_from_disk.append(restored_node)
if invalid_nodes:
self.log.warn(f"We're learning about domain '{self.learning_domain}', but found nodes from other domains; "
self.log.warn(f"We're learning about domain '{self.domain}', but found nodes from other domains; "
f"let's ignore them. These domains and nodes are: {dict(invalid_nodes)}")
return restored_from_disk
@ -821,9 +822,9 @@ class Learner:
self.log.info("Bad response from teacher {}: {} - {}".format(current_teacher, response, response.content))
return
if self.learning_domain != current_teacher.serving_domain:
self.log.debug(f"{current_teacher} is serving '{current_teacher.serving_domain}', "
f"ignore since we are learning about '{self.learning_domain}'")
if self.domain != current_teacher.domain:
self.log.debug(f"{current_teacher} is serving '{current_teacher.domain}', "
f"ignore since we are learning about '{self.domain}'")
return # This node is not serving our domain.
#
@ -946,7 +947,7 @@ class Teacher:
# Fleet
#
self.serving_domain = domain
self.domain = domain
self.fleet_state_checksum = None
self.fleet_state_updated = None
self.last_seen = NEVER_SEEN("No Connection to Node")
@ -1362,7 +1363,7 @@ class Teacher:
"last_seen": last_seen,
"fleet_state": node.fleet_state_checksum or 'unknown',
"fleet_state_icon": fleet_icon,
"domain": node.serving_domain,
"domain": node.domain,
'version': nucypher.__version__
}
return payload

8
nucypher/network/server.py
View File

@ -80,7 +80,7 @@ class ProxyRESTServer:
def make_rest_app(
db_filepath: str,
this_node,
serving_domain,
domain,
log: Logger=Logger("http-application-layer")
) -> Tuple[Flask, Datastore]:
"""
@ -97,12 +97,12 @@ def make_rest_app(
log.info("Starting datastore {}".format(db_filepath))
datastore = Datastore(db_filepath)
rest_app = _make_rest_app(weakref.proxy(datastore), weakref.proxy(this_node), serving_domain, log)
rest_app = _make_rest_app(weakref.proxy(datastore), weakref.proxy(this_node), domain, log)
return rest_app, datastore
def _make_rest_app(datastore: Datastore, this_node, serving_domain: str, log: Logger) -> Flask:
def _make_rest_app(datastore: Datastore, this_node, domain: str, log: Logger) -> Flask:
from nucypher.characters.lawful import Alice, Ursula
_alice_class = Alice
@ -436,7 +436,7 @@ def _make_rest_app(datastore: Datastore, this_node, serving_domain: str, log: Lo
content = status_template.render(this_node=this_node,
known_nodes=this_node.known_nodes,
previous_states=previous_states,
domain=serving_domain,
domain=domain,
version=nucypher.__version__,
checksum_address=this_node.checksum_address)
except Exception as e:

2
nucypher/utilities/prometheus/collector.py
View File

@ -116,7 +116,7 @@ class UrsulaInfoMetricsCollector(BaseMetricsCollector):
base_payload = {'app_version': nucypher.__version__,
'teacher_version': str(self.ursula.TEACHER_VERSION),
'host': str(self.ursula.rest_interface),
'domain': self.ursula.learning_domain,
'domain': self.ursula.domain,
'nickname': str(self.ursula.nickname),
'nickname_icon': self.ursula.nickname_icon,
'fleet_state': str(self.ursula.known_nodes.checksum),

6
tests/acceptance/characters/test_decentralized_grant.py
View File

@ -104,8 +104,8 @@ def test_bob_retrieves_treasure_map_from_decentralized_node(enacted_blockchain_p
except with an `enacted_blockchain_policy`.
"""
bob = enacted_blockchain_policy.bob
_previous_domain = bob.learning_domain
bob.learning_domain = None # Bob has no knowledge of the network.
_previous_domain = bob.domain
bob.domain = None # Bob has no knowledge of the network.
with pytest.raises(bob.NotEnoughTeachers):
treasure_map_from_wire = bob.get_treasure_map(enacted_blockchain_policy.alice.stamp,
@ -113,7 +113,7 @@ def test_bob_retrieves_treasure_map_from_decentralized_node(enacted_blockchain_p
# Bob finds out about one Ursula (in the real world, a seed node, hardcoded based on his learning domain)
bob.done_seeding = False
bob.learning_domain = _previous_domain
bob.domain = _previous_domain
# ...and then learns about the rest of the network.
bob.learn_from_teacher_node(eager=True)

3
tests/acceptance/cli/ursula/test_stake_via_allocation_contract.py
View File

@ -526,7 +526,8 @@ def test_collect_rewards_integration(click_runner,
rest_port=ursula_port,
start_working_now=False,
network_middleware=MockRestMiddleware(),
db_filepath=tempfile.mkdtemp())
db_filepath=tempfile.mkdtemp(),
domain=TEMPORARY_DOMAIN)
MOCK_KNOWN_URSULAS_CACHE[ursula_port] = ursula
assert ursula.worker_address == worker_address

3
tests/acceptance/cli/ursula/test_stakeholder_and_ursula.py
View File

@ -595,7 +595,8 @@ def test_collect_rewards_integration(click_runner,
rest_host='127.0.0.1',
rest_port=ursula_port,
network_middleware=MockRestMiddleware(),
db_filepath=tempfile.mkdtemp())
db_filepath=tempfile.mkdtemp(),
domain=TEMPORARY_DOMAIN)
MOCK_KNOWN_URSULAS_CACHE[ursula_port] = ursula
assert ursula.worker_address == worker_address

3
tests/acceptance/learning/test_fault_tolerance.py
View File

@ -20,6 +20,7 @@ from twisted.logger import LogLevel, globalLogPublisher
from constant_sorrow.constants import NOT_SIGNED
from nucypher.acumen.perception import FleetSensor
from nucypher.config.constants import TEMPORARY_DOMAIN
from nucypher.crypto.powers import TransactingPower
from nucypher.network.nodes import Learner
from tests.utils.middleware import MockRestMiddleware
@ -45,7 +46,7 @@ def test_blockchain_ursula_stamp_verification_tolerance(blockchain_ursulas, mock
unsigned._Teacher__decentralized_identity_evidence = NOT_SIGNED
# Wipe known nodes!
lonely_blockchain_learner._Learner__known_nodes = FleetSensor()
lonely_blockchain_learner._Learner__known_nodes = FleetSensor(domain=TEMPORARY_DOMAIN)
lonely_blockchain_learner._current_teacher_node = blockchain_teacher
lonely_blockchain_learner.remember_node(blockchain_teacher)

3
tests/acceptance/network/test_network_actors.py
View File

@ -23,6 +23,7 @@ import pytest
from nucypher.acumen.nicknames import Nickname
from nucypher.acumen.perception import FleetSensor
from nucypher.characters.unlawful import Vladimir
from nucypher.config.constants import TEMPORARY_DOMAIN
from nucypher.crypto.powers import SigningPower
from nucypher.datastore.models import TreasureMap
from tests.utils.middleware import MockRestMiddleware
@ -44,7 +45,7 @@ def test_all_blockchain_ursulas_know_about_all_other_ursulas(blockchain_ursulas,
def test_blockchain_alice_finds_ursula_via_rest(blockchain_alice, blockchain_ursulas):
# Imagine alice knows of nobody.
blockchain_alice._Learner__known_nodes = FleetSensor()
blockchain_alice._Learner__known_nodes = FleetSensor(domain=TEMPORARY_DOMAIN)
blockchain_alice.remember_node(blockchain_ursulas[0])
blockchain_alice.learn_from_teacher_node()

2
tests/integration/config/test_character_configuration.py
View File

@ -77,7 +77,7 @@ def test_federated_development_character_configurations(character, configuration
assert thing_one.federated_only is True
# Domain
assert TEMPORARY_DOMAIN == thing_one.learning_domain
assert TEMPORARY_DOMAIN == thing_one.domain
# Node Storage
assert configuration.TEMP_CONFIGURATION_DIR_PREFIX in thing_one.keyring_root

3
tests/integration/config/test_configuration_persistence.py
View File

@ -21,6 +21,7 @@ import os
from nucypher.characters.lawful import Bob
from nucypher.config.characters import AliceConfiguration
from nucypher.config.constants import TEMPORARY_DOMAIN
from nucypher.crypto.powers import DecryptingPower, SigningPower
from tests.constants import INSECURE_DEVELOPMENT_PASSWORD
from tests.utils.middleware import MockRestMiddleware
@ -31,7 +32,7 @@ def test_alices_powers_are_persistent(federated_ursulas, tmpdir):
alice_config = AliceConfiguration(
config_root=os.path.join(tmpdir, 'nucypher-custom-alice-config'),
network_middleware=MockRestMiddleware(),
known_nodes=federated_ursulas,
domain=TEMPORARY_DOMAIN,
start_learning_now=False,
federated_only=True,
save_metadata=False,

76
tests/integration/learning/test_domains.py
View File

@ -15,50 +15,50 @@
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from functools import partial
from nucypher.acumen.perception import FleetSensor
from nucypher.config.storages import LocalFileBasedNodeStorage
def test_learner_learns_about_domains_separately(lonely_ursula_maker, caplog):
_lonely_ursula_maker = partial(lonely_ursula_maker, know_each_other=True, quantity=3)
hero_learner, other_first_domain_learner = lonely_ursula_maker(domain="nucypher1.test_suite", quantity=2)
_nobody = lonely_ursula_maker(domain="nucypher1.test_suite", quantity=1).pop()
other_first_domain_learner.remember_node(_nobody)
global_learners = _lonely_ursula_maker(domain="nucypher1.test_suite")
first_domain_learners = _lonely_ursula_maker(domain="nucypher1.test_suite")
second_domain_learners = _lonely_ursula_maker(domain="nucypher2.test_suite")
second_domain_learners = lonely_ursula_maker(domain="nucypher2.test_suite", know_each_other=True, quantity=3)
big_learner = global_learners.pop()
assert len(hero_learner.known_nodes) == 0
assert len(big_learner.known_nodes) == 2
# Learn about the fist domain.
big_learner._current_teacher_node = first_domain_learners.pop()
big_learner.learn_from_teacher_node()
# Learn about the second domain.
big_learner._current_teacher_node = second_domain_learners.pop()
big_learner.learn_from_teacher_node()
# Learn from a teacher in our domain.
hero_learner.remember_node(other_first_domain_learner)
hero_learner.learn_from_teacher_node()
# All domain 1 nodes
assert len(big_learner.known_nodes) == 5
assert len(hero_learner.known_nodes) == 2
new_first_domain_learner = _lonely_ursula_maker(domain="nucypher1.test_suite").pop()
_new_second_domain_learner = _lonely_ursula_maker(domain="nucypher2.test_suite")
# Learn about the second domain.
hero_learner._current_teacher_node = second_domain_learners.pop()
hero_learner.learn_from_teacher_node()
# All domain 1 nodes
assert len(hero_learner.known_nodes) == 2
new_first_domain_learner = lonely_ursula_maker(domain="nucypher1.test_suite", quantity=1).pop()
_new_second_domain_learner = lonely_ursula_maker(domain="nucypher2.test_suite", quantity=1).pop()
new_first_domain_learner.remember_node(hero_learner)
new_first_domain_learner._current_teacher_node = big_learner
new_first_domain_learner.learn_from_teacher_node()
# This node, in the first domain, didn't learn about the second domain.
assert not set(second_domain_learners).intersection(new_first_domain_learner.known_nodes)
# However, it learned about *all* of the nodes in its own domain.
assert set(first_domain_learners).intersection(
n.mature() for n in new_first_domain_learner.known_nodes) == first_domain_learners
assert hero_learner in new_first_domain_learner.known_nodes
assert other_first_domain_learner in new_first_domain_learner.known_nodes
assert _nobody in new_first_domain_learner.known_nodes
def test_learner_restores_metadata_from_storage(lonely_ursula_maker, tmpdir):
# Create a local file-based node storage
root = tmpdir.mkdir("known_nodes")
metadata = root.mkdir("metadata")
@ -82,13 +82,39 @@ def test_learner_restores_metadata_from_storage(lonely_ursula_maker, tmpdir):
know_each_other=True,
save_metadata=False)
learner, buddy = new_learners
buddy._Learner__known_nodes = FleetSensor()
buddy._Learner__known_nodes = FleetSensor(domain="fistro")
# The learner shouldn't learn about any node from the first domain, since it's different.
learner.learn_from_teacher_node()
for restored_node in learner.known_nodes:
assert restored_node.mature().serving_domain == learner.learning_domain
assert restored_node.mature().domain == learner.domain
# In fact, since the storage only contains nodes from a different domain,
# the learner should only know its buddy from the second domain.
assert set(learner.known_nodes) == {buddy}
def test_learner_ignores_stored_nodes_from_other_domains(lonely_ursula_maker, tmpdir):
learner, other_staker = lonely_ursula_maker(domain="call-it-mainnet",
know_each_other=True,
quantity=2)
pest, *other_ursulas_from_the_wrong_side_of_the_tracks = lonely_ursula_maker(domain="i-dunno-testt-maybe",
quantity=5,
know_each_other=True)
assert pest not in learner.known_nodes
pest._current_teacher_node = learner
pest.learn_from_teacher_node()
##################################
# Prior to #2423, learner remembered pest because POSTed node metadata was not domain-checked.
# This is how ibex nodes initially made their way into mainnet fleet states.
assert pest not in learner.known_nodes # But not anymore.
# Once pest made its way into learner, learner taught passed it to other mainnet nodes.
learner.known_nodes._nodes[pest.checksum_address] = pest # This used to happen anyway.
other_staker._current_teacher_node = learner
other_staker.learn_from_teacher_node() # And once it did, the node from the wrong domain spread.
assert pest not in other_staker.known_nodes # But not anymore.

5
tests/integration/learning/test_firstula_circumstances.py
View File

@ -25,12 +25,11 @@ from tests.utils.ursula import make_federated_ursulas
def test_proper_seed_node_instantiation(lonely_ursula_maker):
_lonely_ursula_maker = partial(lonely_ursula_maker, quantity=1)
firstula = _lonely_ursula_maker().pop()
firstula = _lonely_ursula_maker(domain="this-is-meaningful-now").pop()
firstula_as_seed_node = firstula.seed_node_metadata()
any_other_ursula = _lonely_ursula_maker(seed_nodes=[firstula_as_seed_node], domain="useless domain").pop()
any_other_ursula = _lonely_ursula_maker(seed_nodes=[firstula_as_seed_node], domain="this-is-meaningful-now").pop()
assert not any_other_ursula.known_nodes
# print(f"**********************Starting {any_other_ursula} loop")
any_other_ursula.start_learning_loop(now=True)
assert firstula in any_other_ursula.known_nodes

2
tests/integration/learning/test_learning_upgrade.py
View File

@ -33,7 +33,7 @@ def test_emit_warning_upon_new_version(lonely_ursula_maker, caplog):
know_each_other=True)
learner, _bystander = lonely_ursula_maker(quantity=2, domain="no hardcodes")
learner.learning_domain = "no hardcodes"
learner.domain = "no hardcodes"
learner.remember_node(teacher)
teacher.remember_node(learner)
teacher.remember_node(new_node)

2
tests/integration/learning/test_learning_versions.py
View File

@ -101,4 +101,4 @@ def test_deserialize_ursulas_version_2():
assert version == Ursula.LEARNER_VERSION
resurrected_ursula = Ursula.from_bytes(fossilized_ursula, fail_fast=True)
assert TEMPORARY_DOMAIN.encode('utf-8') == resurrected_ursula.domain
assert TEMPORARY_DOMAIN == resurrected_ursula.domain

6
tests/integration/network/test_treasure_map_integration.py
View File

@ -74,8 +74,8 @@ def test_bob_can_retrieve_the_treasure_map_and_decrypt_it(enacted_federated_poli
that Bob can retrieve it with only the information about which he is privy pursuant to the PolicyGroup.
"""
bob = enacted_federated_policy.bob
_previous_domain = bob.learning_domain
bob.learning_domain = None # Bob has no knowledge of the network.
_previous_domain = bob.domain
bob.domain = None # Bob has no knowledge of the network.
# Of course, in the real world, Bob has sufficient information to reconstitute a PolicyGroup, gleaned, we presume,
# through a side-channel with Alice.
@ -88,7 +88,7 @@ def test_bob_can_retrieve_the_treasure_map_and_decrypt_it(enacted_federated_poli
# Bob finds out about one Ursula (in the real world, a seed node, hardcoded based on his learning domain)
bob.done_seeding = False
bob.learning_domain = _previous_domain
bob.domain = _previous_domain
# ...and then learns about the rest of the network.
bob.learn_from_teacher_node(eager=True)

2
tests/mock/performance_mocks.py
View File

@ -197,7 +197,7 @@ class NotARestApp:
if self._actual_rest_app is None:
self._actual_rest_app, self._datastore = make_rest_app(db_filepath=tempfile.mkdtemp(),
this_node=self.this_node,
serving_domain=None)
domain=None)
_new_view_functions = self._ViewFunctions(self._actual_rest_app.view_functions)
self._actual_rest_app.view_functions = _new_view_functions
self._actual_rest_apps.append(

17
tests/unit/test_work_tracker_error_handling.py
View File

@ -1,3 +1,20 @@
"""
This file is part of nucypher.
nucypher is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
nucypher is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import pytest_twisted
from twisted.internet import task
from twisted.internet import threads