From 4c0f2009e7c34c1ae59f48ebcc92fd82b262f776 Mon Sep 17 00:00:00 2001 From: jMyles Date: Fri, 29 Jun 2018 20:13:10 -0700 Subject: [PATCH] Final compartmentalization of serialization logic. Fixes #172. --- nucypher/network/protocols.py | 36 ++++--- nucypher/network/server.py | 55 +++++----- nucypher/policy/models.py | 188 ++++++++++++++++++++++++++-------- 3 files changed, 196 insertions(+), 83 deletions(-) diff --git a/nucypher/network/protocols.py b/nucypher/network/protocols.py index b2b2b9622..c1c899bb4 100644 --- a/nucypher/network/protocols.py +++ b/nucypher/network/protocols.py @@ -54,22 +54,30 @@ class NucypherHashProtocol(KademliaProtocol): self.welcomeIfNewNode(source) self.log.debug("got a store request from %s" % str(sender)) - # TODO: Why is this logic here? This is madness. See #172. - if value.startswith(bytes(constants.BYTESTRING_IS_URSULA_IFACE_INFO)): - header, signature, sender_pubkey_sig,\ - public_address, rest_info, dht_info = ursula_interface_splitter(value) + header, payload = default_constant_splitter(value, return_remainder=True) - # TODO: TTL? - hrac = public_address + rest_info + dht_info - do_store = self.determine_legality_of_dht_key(signature, sender_pubkey_sig, - hrac, key, value) - elif value.startswith(bytes(constants.BYTESTRING_IS_TREASURE_MAP)): - header, signature, sender_pubkey_sig, hrac, message = dht_with_hrac_splitter( - value, return_remainder=True) + if header == constants.BYTESTRING_IS_URSULA_IFACE_INFO: + from nucypher.characters import Ursula + stranger_ursula = Ursula.from_bytes(payload, federated_only=True) # TODO: Is federated_only the right thing here? - # TODO: TTL? - do_store = self.determine_legality_of_dht_key(signature, sender_pubkey_sig, - hrac, key, value) + if stranger_ursula.verify_interface() and key == digest(stranger_ursula.canonical_public_address): + self.sourceNode._node_storage[key] = stranger_ursula # TODO: 340 + return True + else: + self.log.warning("Got request to store invalid node: {} / {}".format(key, value)) + self.illegal_keys_seen.append(key) + return False + elif header == constants.BYTESTRING_IS_TREASURE_MAP: + from nucypher.policy.models import TreasureMap + try: + treasure_map = TreasureMap.from_bytes(payload) + self.log.info("Storing TreasureMap: {} / {}".format(key, value)) + self.sourceNode._treasure_maps[treasure_map.public_id()] = value + return True + except TreasureMap.InvalidPublicSignature: + self.log.warning("Got request to store invalid TreasureMap: {} / {}".format(key, value)) + self.illegal_keys_seen.append(key) + return False else: self.log.info( "Got request to store bad k/v: {} / {}".format(key, value)) diff --git a/nucypher/network/server.py b/nucypher/network/server.py index 0e347a0ee..3c27e4079 100644 --- a/nucypher/network/server.py +++ b/nucypher/network/server.py @@ -6,23 +6,23 @@ from typing import ClassVar import kademlia from apistar import http, Route, App from apistar.http import Response -from bytestring_splitter import VariableLengthBytestring, BytestringSplitter +from constant_sorrow import constants from kademlia.crawling import NodeSpiderCrawl from kademlia.network import Server from kademlia.utils import digest -from umbral import pre -from umbral.fragments import KFrag +from bytestring_splitter import VariableLengthBytestring, BytestringSplitter from nucypher.config.configs import NetworkConfiguration +from nucypher.crypto.constants import PUBLIC_ADDRESS_LENGTH, PUBLIC_KEY_LENGTH from nucypher.crypto.kits import UmbralMessageKit from nucypher.crypto.powers import EncryptingPower, SigningPower from nucypher.keystore.threading import ThreadedSession -from nucypher.network.protocols import NucypherSeedOnlyProtocol, NucypherHashProtocol, \ - dht_with_hrac_splitter, InterfaceInfo +from nucypher.network.protocols import NucypherSeedOnlyProtocol, NucypherHashProtocol, InterfaceInfo from nucypher.network.storage import SeedOnlyStorage +from umbral import pre +from umbral.fragments import KFrag from umbral.keys import UmbralPublicKey from umbral.signing import Signature -from nucypher.crypto.constants import PUBLIC_ADDRESS_LENGTH, PUBLIC_KEY_LENGTH class NucypherDHTServer(Server): @@ -73,6 +73,10 @@ class NucypherDHTServer(Server): loop = asyncio.get_event_loop() return loop.run_until_complete(self.get(bytes(key))) + def set_now(self, key, value): + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.set(key, value)) + async def set(self, key, value): """ Set the given string key to the given value in the network. @@ -91,11 +95,10 @@ class NucypherSeedOnlyDHTServer(NucypherDHTServer): class ProxyRESTServer: - public_information_splitter = BytestringSplitter(Signature, - (UmbralPublicKey, int(PUBLIC_KEY_LENGTH)), - (UmbralPublicKey, int(PUBLIC_KEY_LENGTH)), - int(PUBLIC_ADDRESS_LENGTH)) + (UmbralPublicKey, int(PUBLIC_KEY_LENGTH)), + (UmbralPublicKey, int(PUBLIC_KEY_LENGTH)), + int(PUBLIC_ADDRESS_LENGTH)) def __init__(self, host=None, port=None, db_name=None, *args, **kwargs): self.rest_interface = InterfaceInfo(host=host, port=port) @@ -104,17 +107,17 @@ class ProxyRESTServer: self._rest_app = None @classmethod - def from_config(cls, network_config: NetworkConfiguration=None): + def from_config(cls, network_config: NetworkConfiguration = None): """Create a server object from config values, or from a config file.""" # if network_config is None: - # NetworkConfiguration._load() + # NetworkConfiguration._load() instance = cls() def public_key(self, power_class: ClassVar): """Implemented on Ursula""" raise NotImplementedError - def public_address(self): + def canonical_public_address(self): """Implemented on Ursula""" raise NotImplementedError @@ -138,10 +141,10 @@ class ProxyRESTServer: Route('/consider_arrangement', 'POST', self.consider_arrangement), - Route('/treasure_map/{treasure_map_id_as_hex}', + Route('/treasure_map/{treasure_map_id}', 'GET', self.provide_treasure_map), - Route('/treasure_map/{treasure_map_id_as_hex}', + Route('/treasure_map/{treasure_map_id}', 'POST', self.receive_treasure_map), ] @@ -166,7 +169,6 @@ class ProxyRESTServer: def rest_url(self): return "{}:{}".format(self.ip_address, self.rest_port) - ##################################### # Actual REST Endpoints and utilities ##################################### @@ -178,7 +180,8 @@ class ProxyRESTServer: headers = {'Content-Type': 'application/octet-stream'} # TODO: Calling public_address() works here because this is mixed in with Character, but it's not really right. - message = bytes(self.public_key(SigningPower)) + bytes(self.public_key(EncryptingPower)) + self.public_address + message = bytes(self.public_key(SigningPower)) + bytes( + self.public_key(EncryptingPower)) + self.canonical_public_address signature = self.stamp(message) response = Response( @@ -187,11 +190,11 @@ class ProxyRESTServer: return response - def list_all_active_nodes_about_which_we_know(self): + def list_all_active_nodes_about_which_we_know(self, request: http.Request): headers = {'Content-Type': 'application/octet-stream'} # TODO: mm hmmph *slowly exhales* fffff. Some 227 right here. - ursulas_as_bytes = bytes().join(self.dht_server.protocol.ursulas.values()) - ursulas_as_bytes += self.interface_info_with_metadata() + ursulas_as_bytes = bytes().join(bytes(n) for n in self._known_nodes.values()) + ursulas_as_bytes += bytes(self) signature = self.stamp(ursulas_as_bytes) return Response(bytes(signature) + ursulas_as_bytes, headers=headers) @@ -223,8 +226,6 @@ class ProxyRESTServer: """ hrac = binascii.unhexlify(hrac_as_hex) policy_message_kit = UmbralMessageKit.from_bytes(request.body) - # group_payload_splitter = BytestringSplitter(PublicKey) - # policy_payload_splitter = BytestringSplitter((KFrag, KFRAG_LENGTH)) alice = self._alice_class.from_public_keys({SigningPower: policy_message_kit.sender_pubkey_sig}) @@ -243,10 +244,10 @@ class ProxyRESTServer: with ThreadedSession(self.db_engine) as session: self.datastore.attach_kfrag_to_saved_arrangement( - alice, - hrac_as_hex, - kfrag, - session=session) + alice, + hrac_as_hex, + kfrag, + session=session) return # TODO: Return A 200, with whatever policy metadata. @@ -256,7 +257,7 @@ class ProxyRESTServer: work_order = WorkOrder.from_rest_payload(hrac, request.body) with ThreadedSession(self.db_engine) as session: kfrag_bytes = self.datastore.get_policy_arrangement(hrac.hex().encode(), - session=session).k_frag # Careful! :-) + session=session).k_frag # Careful! :-) # TODO: Push this to a lower level. kfrag = KFrag.from_bytes(kfrag_bytes) cfrag_byte_stream = b"" diff --git a/nucypher/policy/models.py b/nucypher/policy/models.py index cf9bc372d..f07c9de7e 100644 --- a/nucypher/policy/models.py +++ b/nucypher/policy/models.py @@ -161,41 +161,32 @@ class Policy: """ return keccak_digest(bytes(self.alice.stamp) + bytes(self.bob.stamp) + self.label) - def treasure_map_dht_key(self): - """ - We need a key that Bob can glean from knowledge he already has *and* which Ursula can verify came from us. - Ursula will refuse to propagate this key if it she can't prove that our public key, which is included in it, - was used to sign the payload. - - Our public key (which everybody knows) and the hrac above. - """ - return keccak_digest(bytes(self.alice.stamp) + self.hrac()) - - def publish_treasure_map(self, network_middleare=None, use_dht=False): - if network_middleare is None and use_dht is False: - raise ValueError("Can't engage the REST swarm without networky stuff.") - tmap_message_kit, signature_for_bob = self.alice.encrypt_for( - self.bob, - self.treasure_map.packed_payload()) - signature_for_ursula = self.alice.stamp(bytes(self.alice.stamp) + self.hrac()) - - # In order to know this is safe to propagate, Ursula needs to see a signature, our public key, - # and, reasons explained in treasure_map_dht_key above, the uri_hash. - # TODO: Clean this up. See #172. - map_payload = signature_for_ursula + self.alice.stamp + self.alice.public_address + self.hrac() + tmap_message_kit.to_bytes() - map_id = self.treasure_map_dht_key() - + def publish_treasure_map(self, network_middleare): + self.treasure_map.prepare_for_publication(self.bob.public_key(EncryptingPower), + self.bob.public_key(SigningPower), + self.alice.stamp, + self.label + ) if not self.alice._known_nodes: + # TODO: Optionally block. raise RuntimeError("Alice hasn't learned of any nodes. Thus, she can't push the TreasureMap.") - for node in self.alice._known_nodes.values(): - response = network_middleare.push_treasure_map_to_node(node, map_id, - constants.BYTESTRING_IS_TREASURE_MAP + map_payload) - # TODO: Do something here based on success or failure - if response.status_code == 204: - pass + responses = {} - return tmap_message_kit, map_payload, signature_for_bob, signature_for_ursula + for node in self.alice._known_nodes.values(): + # TODO: It's way overkill to push this to every node we know about. Come up with a system. 342 + response = network_middleare.put_treasure_map_on_node(node, + self.treasure_map.public_id(), + bytes(self.treasure_map) + ) + if response.status_code == 202: + responses[node] = response + # TODO: Handle response wherein node already had a copy of this TreasureMap. 341 + else: + # TODO: Do something useful here. + raise RuntimeError + + return responses def publish(self, network_middleware) -> None: """Spread word of this Policy far and wide.""" @@ -313,28 +304,141 @@ class FederatedPolicy(Policy): raise self.MoreKFragsThanArrangements -class TreasureMap(object): - def __init__(self, m, ursula_interface_ids=None): - self.m = m - self.ids = set(ursula_interface_ids or set()) +class TreasureMap: + splitter = BytestringSplitter(Signature, + (bytes, KECCAK_DIGEST_LENGTH), # hrac + (UmbralMessageKit, VariableLengthBytestring) + ) + node_id_splitter = BytestringSplitter(PUBLIC_ADDRESS_LENGTH) - def packed_payload(self): - return msgpack.dumps(self.nodes_as_bytes() + [self.m]) + class InvalidPublicSignature(Exception): + """Raised when the public signature (typically intended for Ursula) is not valid.""" + + def __init__(self, + m=None, + node_ids=None, + message_kit=None, + public_signature=None, + hrac=None): + + if m is not None: + if m > 255: + raise ValueError( + "Largest allowed value for m is 255. Why the heck are you trying to make it larger than that anyway? That's too big.") + self.m = m + self.node_ids = node_ids or set() + else: + self.m = constants.NO_DECRYPTION_PERFORMED + self.node_ids = constants.NO_DECRYPTION_PERFORMED + + self.message_kit = message_kit + self._signature_for_bob = None + self._public_signature = public_signature + self._hrac = hrac + self._payload = None + + def prepare_for_publication(self, bob_encrypting_key, bob_verifying_key, alice_stamp, label): + plaintext = self.m.to_bytes(1, "big") + self.nodes_as_bytes() + + self.message_kit, _signature_for_bob = encrypt_and_sign(bob_encrypting_key, + plaintext=plaintext, + signer=alice_stamp, + ) + """ + Here's our "hashed resource authentication code". + + A hash of: + * Alice's public key + * Bob's public key + * the uri + + Alice and Bob have all the information they need to construct this. + Ursula does not, so we share it with her. + + This way, Bob can generate it and use it to find the TreasureMap. + """ + self._hrac = keccak_digest(bytes(alice_stamp) + bytes(bob_verifying_key) + label) + self._public_signature = alice_stamp(bytes(alice_stamp) + self._hrac) + self._set_payload() + + def _set_payload(self): + self._payload = self._public_signature + self._hrac + bytes( + VariableLengthBytestring(self.message_kit.to_bytes())) + + def __bytes__(self): + if self._payload is None: + self._set_payload() + + return self._payload + + @property + def _verifying_key(self): + return self.message_kit.sender_pubkey_sig def nodes_as_bytes(self): - return [bytes(ursula_id) for ursula_id in self.ids] + if self.node_ids == constants.NO_DECRYPTION_PERFORMED: + return constants.NO_DECRYPTION_PERFORMED + else: + return bytes().join(bytes(ursula_id) for ursula_id in self.node_ids) def add_ursula(self, ursula): - self.ids.add(bytes(ursula.stamp)) + if self.node_ids == constants.NO_DECRYPTION_PERFORMED: + raise TypeError("This TreasureMap is encrypted. You can't add another node without decrypting it.") + self.node_ids.add(ursula.canonical_public_address) + + def public_id(self): + """ + We need an ID that Bob can glean from knowledge he already has *and* which Ursula can verify came from Alice. + Ursula will refuse to propagate this if it she can't prove the payload is signed by Alice's public key, + which is included in it, + """ + return keccak_digest(bytes(self._verifying_key) + bytes(self._hrac)).hex() + + @classmethod + def from_bytes(cls, bytes_representation, verify=True): + signature, hrac, tmap_message_kit = \ + cls.splitter(bytes_representation) + + treasure_map = cls( + message_kit=tmap_message_kit, + public_signature=signature, + hrac=hrac, + ) + + if verify: + treasure_map.public_verify() + + return treasure_map + + def public_verify(self): + message = bytes(self._verifying_key) + self._hrac + verified = self._public_signature.verify(message, self._verifying_key) + + if verified: + return True + else: + raise self.InvalidPublicSignature("This TreasureMap is not properly publicly signed by Alice.") + + def orient(self, compass): + """ + When Bob receives the TreasureMap, he'll pass a compass (a callable which can verify and decrypt the + payload message kit). + """ + verified, map_in_the_clear = compass(message_kit=self.message_kit) + if verified: + self.m = map_in_the_clear[0] + self.node_ids = self.node_id_splitter.repeat(map_in_the_clear[1:], as_set=True) + else: + raise self.InvalidPublicSignature("This TreasureMap does not contain the correct signature from Alice to Bob.") def __eq__(self, other): - return self.ids == other.ids + return self.node_ids == other.node_ids def __iter__(self): - return iter(self.ids) + return iter(self.node_ids) def __len__(self): - return len(self.ids) + return len(self.node_ids) class WorkOrder(object):