Final compartmentalization of serialization logic. Fixes #172.

nucypher/network/protocols.py

@@ -54,22 +54,30 @@ class NucypherHashProtocol(KademliaProtocol):
         self.welcomeIfNewNode(source)
         self.log.debug("got a store request from %s" % str(sender))
 
-        # TODO: Why is this logic here?  This is madness.  See #172.
+        header, payload = default_constant_splitter(value, return_remainder=True)
-        if value.startswith(bytes(constants.BYTESTRING_IS_URSULA_IFACE_INFO)):
-            header, signature, sender_pubkey_sig,\
-            public_address, rest_info, dht_info = ursula_interface_splitter(value)
 
-            # TODO: TTL?
+        if header == constants.BYTESTRING_IS_URSULA_IFACE_INFO:
-            hrac = public_address + rest_info + dht_info
+            from nucypher.characters import Ursula
-            do_store = self.determine_legality_of_dht_key(signature, sender_pubkey_sig,
+            stranger_ursula = Ursula.from_bytes(payload, federated_only=True)  # TODO: Is federated_only the right thing here?
-                                                          hrac, key, value)
-        elif value.startswith(bytes(constants.BYTESTRING_IS_TREASURE_MAP)):
-            header, signature, sender_pubkey_sig, hrac, message = dht_with_hrac_splitter(
-                value, return_remainder=True)
 
-            # TODO: TTL?
+            if stranger_ursula.verify_interface() and key == digest(stranger_ursula.canonical_public_address):
-            do_store = self.determine_legality_of_dht_key(signature, sender_pubkey_sig,
+                self.sourceNode._node_storage[key] = stranger_ursula  # TODO: 340
-                                                          hrac, key, value)
+                return True
+            else:
+                self.log.warning("Got request to store invalid node: {} / {}".format(key, value))
+                self.illegal_keys_seen.append(key)
+                return False
+        elif header == constants.BYTESTRING_IS_TREASURE_MAP:
+            from nucypher.policy.models import TreasureMap
+            try:
+                treasure_map = TreasureMap.from_bytes(payload)
+                self.log.info("Storing TreasureMap: {} / {}".format(key, value))
+                self.sourceNode._treasure_maps[treasure_map.public_id()] = value
+                return True
+            except TreasureMap.InvalidPublicSignature:
+                self.log.warning("Got request to store invalid TreasureMap: {} / {}".format(key, value))
+                self.illegal_keys_seen.append(key)
+                return False
         else:
             self.log.info(
                 "Got request to store bad k/v: {} / {}".format(key, value))

nucypher/network/server.py

@@ -6,23 +6,23 @@ from typing import ClassVar
 import kademlia
 from apistar import http, Route, App
 from apistar.http import Response
-from bytestring_splitter import VariableLengthBytestring, BytestringSplitter
+from constant_sorrow import constants
 from kademlia.crawling import NodeSpiderCrawl
 from kademlia.network import Server
 from kademlia.utils import digest
-from umbral import pre
-from umbral.fragments import KFrag
 
+from bytestring_splitter import VariableLengthBytestring, BytestringSplitter
 from nucypher.config.configs import NetworkConfiguration
+from nucypher.crypto.constants import PUBLIC_ADDRESS_LENGTH, PUBLIC_KEY_LENGTH
 from nucypher.crypto.kits import UmbralMessageKit
 from nucypher.crypto.powers import EncryptingPower, SigningPower
 from nucypher.keystore.threading import ThreadedSession
-from nucypher.network.protocols import NucypherSeedOnlyProtocol, NucypherHashProtocol, \
+from nucypher.network.protocols import NucypherSeedOnlyProtocol, NucypherHashProtocol, InterfaceInfo
-    dht_with_hrac_splitter, InterfaceInfo
 from nucypher.network.storage import SeedOnlyStorage
+from umbral import pre
+from umbral.fragments import KFrag
 from umbral.keys import UmbralPublicKey
 from umbral.signing import Signature
-from nucypher.crypto.constants import PUBLIC_ADDRESS_LENGTH, PUBLIC_KEY_LENGTH
 
 
 class NucypherDHTServer(Server):
@@ -73,6 +73,10 @@ class NucypherDHTServer(Server):
         loop = asyncio.get_event_loop()
         return loop.run_until_complete(self.get(bytes(key)))
 
+    def set_now(self, key, value):
+        loop = asyncio.get_event_loop()
+        return loop.run_until_complete(self.set(key, value))
+
     async def set(self, key, value):
         """
         Set the given string key to the given value in the network.
@@ -91,11 +95,10 @@ class NucypherSeedOnlyDHTServer(NucypherDHTServer):
 
 
 class ProxyRESTServer:
-
     public_information_splitter = BytestringSplitter(Signature,
-                                      (UmbralPublicKey, int(PUBLIC_KEY_LENGTH)),
+                                                     (UmbralPublicKey, int(PUBLIC_KEY_LENGTH)),
-                                      (UmbralPublicKey, int(PUBLIC_KEY_LENGTH)),
+                                                     (UmbralPublicKey, int(PUBLIC_KEY_LENGTH)),
-                                      int(PUBLIC_ADDRESS_LENGTH))
+                                                     int(PUBLIC_ADDRESS_LENGTH))
 
     def __init__(self, host=None, port=None, db_name=None, *args, **kwargs):
         self.rest_interface = InterfaceInfo(host=host, port=port)
@@ -104,17 +107,17 @@ class ProxyRESTServer:
         self._rest_app = None
 
     @classmethod
-    def from_config(cls, network_config: NetworkConfiguration=None):
+    def from_config(cls, network_config: NetworkConfiguration = None):
         """Create a server object from config values, or from a config file."""
         # if network_config is None:
-            # NetworkConfiguration._load()
+        # NetworkConfiguration._load()
         instance = cls()
 
     def public_key(self, power_class: ClassVar):
         """Implemented on Ursula"""
         raise NotImplementedError
 
-    def public_address(self):
+    def canonical_public_address(self):
         """Implemented on Ursula"""
         raise NotImplementedError
 
@@ -138,10 +141,10 @@ class ProxyRESTServer:
             Route('/consider_arrangement',
                   'POST',
                   self.consider_arrangement),
-            Route('/treasure_map/{treasure_map_id_as_hex}',
+            Route('/treasure_map/{treasure_map_id}',
                   'GET',
                   self.provide_treasure_map),
-            Route('/treasure_map/{treasure_map_id_as_hex}',
+            Route('/treasure_map/{treasure_map_id}',
                   'POST',
                   self.receive_treasure_map),
         ]
@@ -166,7 +169,6 @@ class ProxyRESTServer:
     def rest_url(self):
         return "{}:{}".format(self.ip_address, self.rest_port)
 
-
     #####################################
     # Actual REST Endpoints and utilities
     #####################################
@@ -178,7 +180,8 @@ class ProxyRESTServer:
 
         headers = {'Content-Type': 'application/octet-stream'}
         # TODO: Calling public_address() works here because this is mixed in with Character, but it's not really right.
-        message = bytes(self.public_key(SigningPower)) + bytes(self.public_key(EncryptingPower)) + self.public_address
+        message = bytes(self.public_key(SigningPower)) + bytes(
+            self.public_key(EncryptingPower)) + self.canonical_public_address
         signature = self.stamp(message)
 
         response = Response(
@@ -187,11 +190,11 @@ class ProxyRESTServer:
 
         return response
 
-    def list_all_active_nodes_about_which_we_know(self):
+    def list_all_active_nodes_about_which_we_know(self, request: http.Request):
         headers = {'Content-Type': 'application/octet-stream'}
         # TODO: mm hmmph *slowly exhales* fffff.  Some 227 right here.
-        ursulas_as_bytes = bytes().join(self.dht_server.protocol.ursulas.values())
+        ursulas_as_bytes = bytes().join(bytes(n) for n in self._known_nodes.values())
-        ursulas_as_bytes += self.interface_info_with_metadata()
+        ursulas_as_bytes += bytes(self)
         signature = self.stamp(ursulas_as_bytes)
         return Response(bytes(signature) + ursulas_as_bytes, headers=headers)
 
@@ -223,8 +226,6 @@ class ProxyRESTServer:
         """
         hrac = binascii.unhexlify(hrac_as_hex)
         policy_message_kit = UmbralMessageKit.from_bytes(request.body)
-        # group_payload_splitter = BytestringSplitter(PublicKey)
-        # policy_payload_splitter = BytestringSplitter((KFrag, KFRAG_LENGTH))
 
         alice = self._alice_class.from_public_keys({SigningPower: policy_message_kit.sender_pubkey_sig})
 
@@ -243,10 +244,10 @@ class ProxyRESTServer:
 
         with ThreadedSession(self.db_engine) as session:
             self.datastore.attach_kfrag_to_saved_arrangement(
-                                               alice,
+                alice,
-                                               hrac_as_hex,
+                hrac_as_hex,
-                                               kfrag,
+                kfrag,
-                                               session=session)
+                session=session)
 
         return  # TODO: Return A 200, with whatever policy metadata.
 
@@ -256,7 +257,7 @@ class ProxyRESTServer:
         work_order = WorkOrder.from_rest_payload(hrac, request.body)
         with ThreadedSession(self.db_engine) as session:
             kfrag_bytes = self.datastore.get_policy_arrangement(hrac.hex().encode(),
-                                                             session=session).k_frag  # Careful!  :-)
+                                                                session=session).k_frag  # Careful!  :-)
         # TODO: Push this to a lower level.
         kfrag = KFrag.from_bytes(kfrag_bytes)
         cfrag_byte_stream = b""

nucypher/policy/models.py

@@ -161,41 +161,32 @@ class Policy:
         """
         return keccak_digest(bytes(self.alice.stamp) + bytes(self.bob.stamp) + self.label)
 
-    def treasure_map_dht_key(self):
+    def publish_treasure_map(self, network_middleare):
-        """
+        self.treasure_map.prepare_for_publication(self.bob.public_key(EncryptingPower),
-        We need a key that Bob can glean from knowledge he already has *and* which Ursula can verify came from us.
+                                                  self.bob.public_key(SigningPower),
-        Ursula will refuse to propagate this key if it she can't prove that our public key, which is included in it,
+                                                  self.alice.stamp,
-        was used to sign the payload.
+                                                  self.label
-
+                                                  )
-        Our public key (which everybody knows) and the hrac above.
-        """
-        return keccak_digest(bytes(self.alice.stamp) + self.hrac())
-
-    def publish_treasure_map(self, network_middleare=None, use_dht=False):
-        if network_middleare is None and use_dht is False:
-            raise ValueError("Can't engage the REST swarm without networky stuff.")
-        tmap_message_kit, signature_for_bob = self.alice.encrypt_for(
-            self.bob,
-            self.treasure_map.packed_payload())
-        signature_for_ursula = self.alice.stamp(bytes(self.alice.stamp) + self.hrac())
-
-        # In order to know this is safe to propagate, Ursula needs to see a signature, our public key,
-        # and, reasons explained in treasure_map_dht_key above, the uri_hash.
-        # TODO: Clean this up.  See #172.
-        map_payload = signature_for_ursula + self.alice.stamp + self.alice.public_address + self.hrac() + tmap_message_kit.to_bytes()
-        map_id = self.treasure_map_dht_key()
-
         if not self.alice._known_nodes:
+            # TODO: Optionally block.
             raise RuntimeError("Alice hasn't learned of any nodes.  Thus, she can't push the TreasureMap.")
 
-        for node in self.alice._known_nodes.values():
+        responses = {}
-            response = network_middleare.push_treasure_map_to_node(node, map_id,
-                                                                   constants.BYTESTRING_IS_TREASURE_MAP + map_payload)
-            # TODO: Do something here based on success or failure
-            if response.status_code == 204:
-                pass
 
-        return tmap_message_kit, map_payload, signature_for_bob, signature_for_ursula
+        for node in self.alice._known_nodes.values():
+            # TODO: It's way overkill to push this to every node we know about.  Come up with a system.  342
+            response = network_middleare.put_treasure_map_on_node(node,
+                                                                  self.treasure_map.public_id(),
+                                                                  bytes(self.treasure_map)
+                                                                  )
+            if response.status_code == 202:
+                responses[node] = response
+                # TODO: Handle response wherein node already had a copy of this TreasureMap.  341
+            else:
+                # TODO: Do something useful here.
+                raise RuntimeError
+
+        return responses
 
     def publish(self, network_middleware) -> None:
         """Spread word of this Policy far and wide."""
@@ -313,28 +304,141 @@ class FederatedPolicy(Policy):
             raise self.MoreKFragsThanArrangements
 
 
-class TreasureMap(object):
+class TreasureMap:
-    def __init__(self, m, ursula_interface_ids=None):
+    splitter = BytestringSplitter(Signature,
-        self.m = m
+                                  (bytes, KECCAK_DIGEST_LENGTH),  # hrac
-        self.ids = set(ursula_interface_ids or set())
+                                  (UmbralMessageKit, VariableLengthBytestring)
+                                  )
+    node_id_splitter = BytestringSplitter(PUBLIC_ADDRESS_LENGTH)
 
-    def packed_payload(self):
+    class InvalidPublicSignature(Exception):
-        return msgpack.dumps(self.nodes_as_bytes() + [self.m])
+        """Raised when the public signature (typically intended for Ursula) is not valid."""
+
+    def __init__(self,
+                 m=None,
+                 node_ids=None,
+                 message_kit=None,
+                 public_signature=None,
+                 hrac=None):
+
+        if m is not None:
+            if m > 255:
+                raise ValueError(
+                    "Largest allowed value for m is 255.  Why the heck are you trying to make it larger than that anyway?  That's too big.")
+            self.m = m
+            self.node_ids = node_ids or set()
+        else:
+            self.m = constants.NO_DECRYPTION_PERFORMED
+            self.node_ids = constants.NO_DECRYPTION_PERFORMED
+
+        self.message_kit = message_kit
+        self._signature_for_bob = None
+        self._public_signature = public_signature
+        self._hrac = hrac
+        self._payload = None
+
+    def prepare_for_publication(self, bob_encrypting_key, bob_verifying_key, alice_stamp, label):
+        plaintext = self.m.to_bytes(1, "big") + self.nodes_as_bytes()
+
+        self.message_kit, _signature_for_bob = encrypt_and_sign(bob_encrypting_key,
+                                                                plaintext=plaintext,
+                                                                signer=alice_stamp,
+                                                                )
+        """
+        Here's our "hashed resource authentication code".
+
+        A hash of:
+        * Alice's public key
+        * Bob's public key
+        * the uri
+
+        Alice and Bob have all the information they need to construct this.
+        Ursula does not, so we share it with her.
+        
+        This way, Bob can generate it and use it to find the TreasureMap.
+        """
+        self._hrac = keccak_digest(bytes(alice_stamp) + bytes(bob_verifying_key) + label)
+        self._public_signature = alice_stamp(bytes(alice_stamp) + self._hrac)
+        self._set_payload()
+
+    def _set_payload(self):
+        self._payload = self._public_signature + self._hrac + bytes(
+            VariableLengthBytestring(self.message_kit.to_bytes()))
+
+    def __bytes__(self):
+        if self._payload is None:
+            self._set_payload()
+
+        return self._payload
+
+    @property
+    def _verifying_key(self):
+        return self.message_kit.sender_pubkey_sig
 
     def nodes_as_bytes(self):
-        return [bytes(ursula_id) for ursula_id in self.ids]
+        if self.node_ids == constants.NO_DECRYPTION_PERFORMED:
+            return constants.NO_DECRYPTION_PERFORMED
+        else:
+            return bytes().join(bytes(ursula_id) for ursula_id in self.node_ids)
 
     def add_ursula(self, ursula):
-        self.ids.add(bytes(ursula.stamp))
+        if self.node_ids == constants.NO_DECRYPTION_PERFORMED:
+            raise TypeError("This TreasureMap is encrypted.  You can't add another node without decrypting it.")
+        self.node_ids.add(ursula.canonical_public_address)
+
+    def public_id(self):
+        """
+        We need an ID that Bob can glean from knowledge he already has *and* which Ursula can verify came from Alice.
+        Ursula will refuse to propagate this if it she can't prove the payload is signed by Alice's public key,
+        which is included in it,
+        """
+        return keccak_digest(bytes(self._verifying_key) + bytes(self._hrac)).hex()
+
+    @classmethod
+    def from_bytes(cls, bytes_representation, verify=True):
+        signature, hrac, tmap_message_kit = \
+            cls.splitter(bytes_representation)
+
+        treasure_map = cls(
+            message_kit=tmap_message_kit,
+            public_signature=signature,
+            hrac=hrac,
+        )
+
+        if verify:
+            treasure_map.public_verify()
+
+        return treasure_map
+
+    def public_verify(self):
+        message = bytes(self._verifying_key) + self._hrac
+        verified = self._public_signature.verify(message, self._verifying_key)
+
+        if verified:
+            return True
+        else:
+            raise self.InvalidPublicSignature("This TreasureMap is not properly publicly signed by Alice.")
+
+    def orient(self, compass):
+        """
+        When Bob receives the TreasureMap, he'll pass a compass (a callable which can verify and decrypt the
+        payload message kit).
+        """
+        verified, map_in_the_clear = compass(message_kit=self.message_kit)
+        if verified:
+            self.m = map_in_the_clear[0]
+            self.node_ids = self.node_id_splitter.repeat(map_in_the_clear[1:], as_set=True)
+        else:
+            raise self.InvalidPublicSignature("This TreasureMap does not contain the correct signature from Alice to Bob.")
 
     def __eq__(self, other):
-        return self.ids == other.ids
+        return self.node_ids == other.node_ids
 
     def __iter__(self):
-        return iter(self.ids)
+        return iter(self.node_ids)
 
     def __len__(self):
-        return len(self.ids)
+        return len(self.node_ids)
 
 
 class WorkOrder(object):