Merge pull request #116 from jMyles/bytesplitting

Bytesplitting

nkms/characters.py

@@ -1,17 +1,17 @@
 import asyncio
 
 import msgpack
+
 from kademlia.network import Server
 from kademlia.utils import digest
-
 from nkms.crypto import api as API
 from nkms.crypto.api import secure_random, keccak_digest
 from nkms.crypto.constants import NOT_SIGNED, NO_DECRYPTION_PERFORMED
 from nkms.crypto.powers import CryptoPower, SigningPower, EncryptingPower
-from nkms.crypto.utils import verify
 from nkms.keystore.keypairs import Keypair
 from nkms.network import blockchain_client
 from nkms.network.blockchain_client import list_all_ursulas
+from nkms.network.protocols import dht_value_splitter
 from nkms.network.server import NuCypherDHTServer, NuCypherSeedOnlyDHTServer
 from nkms.policy.constants import NOT_FROM_ALICE
 
@@ -136,7 +136,7 @@ class Character(object):
 
         actor = self._lookup_actor(actor_whom_sender_claims_to_be)
 
-        return verify(signature, message, actor.seal), cleartext
+        return signature.verify(message, actor.seal), cleartext
 
     def _lookup_actor(self, actor: "Character"):
         try:
@@ -181,14 +181,12 @@ class Alice(Character):
     def publish_treasure_map(self, policy_group):
         encrypted_treasure_map, signature_for_bob = self.encrypt_for(policy_group.bob,
                                                                      policy_group.treasure_map.packed_payload())
-        signature_for_ursula = self.seal(
+        signature_for_ursula = self.seal(policy_group.hrac())  # TODO: Great use-case for Ciphertext class
-            msgpack.dumps(encrypted_treasure_map))  # TODO: Great use-case for Ciphertext class
 
         # In order to know this is safe to propagate, Ursula needs to see a signature, our public key,
         # and, reasons explained in treasure_map_dht_key above, the uri_hash.
-        dht_value = msgpack.dumps(
+        dht_value = signature_for_ursula + self.seal + policy_group.hrac() + msgpack.dumps(
-            (bytes(signature_for_ursula), bytes(self.seal), policy_group.hrac(),
+            encrypted_treasure_map)  # TODO: Ideally, this is a Ciphertext object instead of msgpack (see #112)
-             encrypted_treasure_map))  # TODO: #114
         dht_key = policy_group.treasure_map_dht_key()
 
         setter = self.server.set(dht_key, b"trmap" + dht_value)
@@ -223,9 +221,10 @@ class Bob(Character):
             getter = self.server.get(ursula_interface_id)
             loop = asyncio.get_event_loop()
             value = loop.run_until_complete(getter)
-            signature, ursula_pubkey_sig, ttl, interface_info = msgpack.loads(
+            signature, ursula_pubkey_sig, hrac, (port, interface, ttl) = dht_value_splitter(value.lstrip(b"uaddr"),
-                value.lstrip(b"uaddr"))  # TODO: If we're going to implement TTL, it'll be here.
+                                                                                            msgpack_remainder=True)
-            port, interface = msgpack.loads(interface_info)
+
+            # TODO: If we're going to implement TTL, it will be here.
             self._ursulas[ursula_interface_id] = Ursula.as_discovered_on_network(port=port, interface=interface,
                                                                                  pubkey_sig_bytes=ursula_pubkey_sig)
 
@@ -236,8 +235,8 @@ class Bob(Character):
         ursula_coro = self.server.get(dht_key)
         event_loop = asyncio.get_event_loop()
         packed_encrypted_treasure_map = event_loop.run_until_complete(ursula_coro)
-        _signature_for_ursula, pubkey_sig_alice, uri_hash, encrypted_treasure_map = msgpack.loads(
+        _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = dht_value_splitter(
-            packed_encrypted_treasure_map[5::])
+            packed_encrypted_treasure_map[5::], msgpack_remainder=True)
         verified, packed_node_list = self.verify_from(self.alice, signature, encrypted_treasure_map,
                                                       signature_is_on_cleartext=True, decrypt=True)
         if not verified:
@@ -276,16 +275,17 @@ class Ursula(Character):
         return self.server.listen(port, interface)
 
     def interface_info(self):
-        return msgpack.dumps((self.port, self.interface))
+        return self.port, self.interface, self.interface_ttl
 
     def interface_dht_key(self):
-        return keccak_digest(bytes(self.seal) + bytes(self.interface_ttl))
+        return self.hash(self.seal + self.interface_hrac())
 
     def interface_dht_value(self):
-        signature = self.seal(self.interface_info())
+        signature = self.seal(self.interface_hrac())
-        ttl = 0  # TODO: We don't actually need this - and it's not currently implemented in a meaningful way,
+        return b"uaddr" + signature + self.seal + self.interface_hrac() + msgpack.dumps(self.interface_info())
-        # but it matches the schema for a shared TreasureMap.  Maybe we use it to indicate a TTL?
+
-        return b"uaddr" + msgpack.dumps((bytes(signature), bytes(self.seal), ttl, self.interface_info()))
+    def interface_hrac(self):
+        return self.hash(msgpack.dumps(self.interface_info()))
 
     def publish_interface_information(self):
         if not self.port and self.interface:
@@ -322,6 +322,15 @@ class Seal(object):
     def __eq__(self, other):
         return other == self._as_tuple() or other == bytes(self)
 
+    def __add__(self, other):
+        return bytes(self) + other
+
+    def __radd__(self, other):
+        return other + bytes(self)
+
+    def __len__(self):
+        return len(bytes(self))
+
 
 class StrangerSeal(Seal):
     """

nkms/crypto/constants.py

@@ -1,2 +1,6 @@
+# TODO: Turn these into classes?
+PUBKEY_SIG_LENGTH = 64
+HASH_DIGEST_LENGTH = 32
+
 NOT_SIGNED = 445
 NO_DECRYPTION_PERFORMED = 455

nkms/crypto/powers.py

@@ -77,7 +77,7 @@ class CryptoPower(object):
             raise NoSigningPower(e)
         msg_digest = b"".join(API.keccak_digest(m) for m in messages)
 
-        return Signature(sig=sig_keypair.sign(msg_digest))
+        return Signature(sig_keypair.sign(msg_digest))
 
     def decrypt(self, ciphertext):
         try:

nkms/crypto/signature.py

@@ -1,13 +1,13 @@
 from nkms.crypto import api as API
 
 
-class Signature(object):
+class Signature(bytes):
     """
     The Signature object allows signatures to be made and verified.
     """
-    LENGTH = 32
+    _EXPECTED_LENGTH = 65
 
-    def __init__(self, v: int=None, r: int=None, s: int=None, sig: bytes=None):
+    def __init__(self, sig_as_bytes: bytes=None, v: int=None, r: int=None, s: int=None):
         """
         Initializes a Signature object.
 
@@ -17,13 +17,23 @@ class Signature(object):
 
         :return: Signature object
         """
-        if sig:
+        if sig_as_bytes and any((v, r, s)):
-            v, r, s = API.ecdsa_load_sig(sig)
+            raise ValueError("Pass *either* sig_as_bytes *or* v, r, and s - don't pass both.")
+        if not any ((sig_as_bytes, v, r, s)):
+            raise ValueError("Pass either sig_as_bytes or v, r, and s.")
+
+        if sig_as_bytes:
+            if not len(sig_as_bytes) == self._EXPECTED_LENGTH:
+                raise ValueError("{} must be {} bytes.".format(self.__class__.__name__, self._EXPECTED_LENGTH))
+            v, r, s = API.ecdsa_load_sig(sig_as_bytes)
 
         self._v = v
         self._r = r
         self._s = s
 
+    def __repr__(self):
+        return "{} v{}: {} - {}".format(__class__.__name__, self._v, self._r, self._s)
+
     def verify(self, message: bytes, pubkey: bytes) -> bool:
         """
         Verifies that a message's signature was valid.

nkms/crypto/utils.py

@@ -1,7 +1,47 @@
-from nkms.crypto import api
+import msgpack
 
 
-def verify(signature: bytes, message: bytes, pubkey: bytes) -> bool:
+class BytestringSplitter(object):
-    msg_digest = api.keccak_digest(message)
+    def __init__(self, *message_types):
-    ecdsa_sig = api.ecdsa_load_sig(bytes(signature))
+        """
-    return api.ecdsa_verify(*ecdsa_sig, msg_digest, pubkey)
+        :param message_types:  A collection of types of messages to parse.
+        """
+        self.message_types = message_types
+
+    def __call__(self, splittable, return_remainder=False, msgpack_remainder=False):
+        if not any((return_remainder, msgpack_remainder)) and len(self) != len(splittable):
+            raise ValueError(
+                "Wrong number of bytes to constitute message types {} - need {}, got {} \n Did you mean to return the remainder?".format(
+                    self.message_types, len(self), len(splittable)))
+        if len(self) > len(splittable):
+            raise ValueError(
+                "Not enough bytes to constitute message types {} - need {}, got {}".format(self.message_types,
+                                                                                           len(self),
+                                                                                           len(splittable)))
+        cursor = 0
+        message_objects = []
+
+        for message_type in self.message_types:
+            message_class, message_length = self.get_message_meta(message_type)
+            expected_end_of_object_bytes = cursor + message_length
+            bytes_for_this_object = splittable[cursor:expected_end_of_object_bytes]
+            message = message_class(bytes_for_this_object)
+
+            message_objects.append(message)
+            cursor = expected_end_of_object_bytes
+
+        remainder = splittable[cursor:]
+
+        if msgpack_remainder:
+            message_objects.append(msgpack.loads(remainder))
+        elif return_remainder:
+            message_objects.append(remainder)
+
+        return message_objects
+
+    def __len__(self):
+        return sum(self.get_message_meta(m)[1] for m in self.message_types)
+
+    @staticmethod
+    def get_message_meta(message_type):
+        return message_type if isinstance(message_type, tuple) else (message_type, message_type._EXPECTED_LENGTH)

nkms/network/protocols.py

@@ -3,12 +3,16 @@ import msgpack
 from kademlia.node import Node
 from kademlia.protocol import KademliaProtocol
 from kademlia.utils import digest
-from nkms.crypto import utils
 from nkms.crypto.api import keccak_digest
+from nkms.crypto.constants import PUBKEY_SIG_LENGTH, HASH_DIGEST_LENGTH
+from nkms.crypto.signature import Signature
+from nkms.crypto.utils import BytestringSplitter
 from nkms.network.constants import NODE_HAS_NO_STORAGE
 from nkms.network.node import NuCypherNode
 from nkms.network.routing import NuCypherRoutingTable
 
+dht_value_splitter = BytestringSplitter(Signature, (bytes, PUBKEY_SIG_LENGTH), (bytes, HASH_DIGEST_LENGTH))
+
 
 class NuCypherHashProtocol(KademliaProtocol):
     def __init__(self, sourceNode, storage, ksize, *args, **kwargs):
@@ -39,16 +43,16 @@ class NuCypherHashProtocol(KademliaProtocol):
         else:
             return NODE_HAS_NO_STORAGE, False
 
-    def determine_legality_of_dht_key(self, signature, sender_pubkey_sig, message, extra_info, dht_key, dht_value):
+    def determine_legality_of_dht_key(self, signature, sender_pubkey_sig, message, hrac, dht_key, dht_value):
-        proper_key = digest(keccak_digest(bytes(sender_pubkey_sig) + bytes(extra_info)))
+        proper_key = digest(keccak_digest(bytes(sender_pubkey_sig) + bytes(hrac)))
 
         # TODO: This try block is not the right approach - a Ciphertext class can resolve this instead.
         try:
             # Ursula uaddr scenario
-            verified = utils.verify(signature, message, sender_pubkey_sig)
+            verified = signature.verify(hrac, sender_pubkey_sig)
         except Exception as e:
             # trmap scenario
-            verified = utils.verify(signature, msgpack.dumps(message), sender_pubkey_sig)
+            verified = signature.verify(msgpack.dumps(message), sender_pubkey_sig)
 
         if not verified or not proper_key == dht_key:
             self.log.warning("Got request to store illegal k/v: {} / {}".format(dht_key, dht_value))
@@ -63,10 +67,11 @@ class NuCypherHashProtocol(KademliaProtocol):
         self.log.debug("got a store request from %s" % str(sender))
 
         if value.startswith(b"uaddr") or value.startswith(b"trmap"):
-            signature, sender_pubkey_sig, extra_info, message = msgpack.loads(value[5::])  # TODO: #114
+            signature, sender_pubkey_sig, hrac, message = dht_value_splitter(value[5::], return_remainder=True)
+
             # extra_info is a hash of the policy_group.id in the case of a treasure map, or a TTL in the case
             # of an Ursula interface.  TODO: Decide whether to keep this notion and, if so, use the TTL.
-            do_store = self.determine_legality_of_dht_key(signature, sender_pubkey_sig, message, extra_info, key, value)
+            do_store = self.determine_legality_of_dht_key(signature, sender_pubkey_sig, message, hrac, key, value)
         else:
             self.log.info("Got request to store bad k/v: {} / {}".format(key, value))
             do_store = False

tests/crypto/test_bytestring_types.py

@@ -0,0 +1,37 @@
+import pytest
+
+from nkms.crypto.api import secure_random
+from nkms.crypto.signature import Signature
+from nkms.crypto.utils import BytestringSplitter
+
+
+def test_split_two_signatures():
+    """
+    We make two random Signatures and concat them.  Then split them and show that we got the proper result.
+    """
+    sig1, sig2 = Signature(secure_random(65)), Signature(secure_random(65))
+    sigs_concatted = sig1 + sig2
+    two_signature_splitter = BytestringSplitter(Signature, Signature)
+    rebuilt_sig1, rebuilt_sig2 = two_signature_splitter(sigs_concatted)
+    assert (sig1, sig2) == (rebuilt_sig1, rebuilt_sig2)
+
+
+def test_split_signature_from_arbitrary_bytes():
+    how_many_bytes = 10
+    signature = Signature(secure_random(65))
+    some_bytes = secure_random(how_many_bytes)
+    splitter = BytestringSplitter(Signature, (bytes, how_many_bytes))
+
+
+    rebuilt_signature, rebuilt_bytes = splitter(signature + some_bytes)
+
+
+def test_trying_to_extract_too_many_bytes_raises_typeerror():
+    how_many_bytes = 10
+    too_many_bytes = 11
+    signature = Signature(secure_random(65))
+    some_bytes = secure_random(how_many_bytes)
+    splitter = BytestringSplitter(Signature, (bytes, too_many_bytes))
+
+    with pytest.raises(ValueError):
+        rebuilt_signature, rebuilt_bytes = splitter(signature + some_bytes, return_remainder=True)

tests/network/test_network_actors.py

@@ -6,7 +6,11 @@ import pytest
 
 from kademlia.utils import digest
 from nkms.characters import Ursula, Alice, Character, Bob, congregate
+from nkms.crypto.constants import PUBKEY_SIG_LENGTH, HASH_DIGEST_LENGTH
+from nkms.crypto.signature import Signature
+from nkms.crypto.utils import BytestringSplitter
 from nkms.network.blockchain_client import list_all_ursulas
+from nkms.network.protocols import dht_value_splitter
 from nkms.policy.constants import NON_PAYMENT
 from nkms.policy.models import PolicyManagerForAlice, PolicyOffer, Policy
 from tests.test_utilities import make_fake_ursulas, MockNetworkyStuff
@@ -57,9 +61,7 @@ def test_vladimir_illegal_interface_key_does_not_propagate():
     assert ursula.server.protocol.illegal_keys_seen == []
 
     # Vladimir does almost everything right....
-    interface_info = msgpack.dumps((vladimir.port, vladimir.interface))
+    value = vladimir.interface_dht_value()
-    signature = vladimir.seal(interface_info)
-    value = b"uaddr" + msgpack.dumps([bytes(signature), bytes(vladimir.seal), 0, interface_info])  # TODO: #114
 
     # Except he sets an illegal key for his interface.
     illegal_key = "Not allowed to set arbitrary key for this."
@@ -111,8 +113,8 @@ def test_alice_finds_ursula():
     getter = ALICE.server.get(all_ursulas[ursula_index])
     loop = asyncio.get_event_loop()
     value = loop.run_until_complete(getter)
-    signature, ursula_pubkey_sig, ttl, interface_info = msgpack.loads(value.lstrip(b"uaddr-"))
+    _signature, _ursula_pubkey_sig, _hrac, interface_info = dht_value_splitter(value.lstrip(b"uaddr-"), return_remainder=True)
-    port, interface = msgpack.loads(interface_info)
+    port = msgpack.loads(interface_info)[0]
     assert port == URSULA_PORT + ursula_index
 
 
@@ -185,9 +187,11 @@ def test_treasure_map_stored_by_ursula_is_the_correct_one_for_bob():
     """
     The TreasureMap given by Alice to Ursula is the correct one for Bob; he can decrypt and read it.
     """
+
     treasure_map_as_set_on_network, signature, policy_group = test_alice_sets_treasure_map_on_network()
-    _signature_for_ursula, pubkey_sig_alice, uri_hash, encrypted_treasure_map = msgpack.loads(
+    _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = dht_value_splitter(
-        treasure_map_as_set_on_network[5::])  # 5:: to account for prepended "trmap"
+        treasure_map_as_set_on_network[5::], msgpack_remainder=True)  # 5:: to account for prepended "trmap"
+
     verified, treasure_map_as_decrypted_by_bob = BOB.verify_from(ALICE, signature,
                                                                  encrypted_treasure_map,
                                                                  decrypt=True,
@@ -222,6 +226,6 @@ def test_treaure_map_is_legit():
         getter = ALICE.server.get(ursula_interface_id)
         loop = asyncio.get_event_loop()
         value = loop.run_until_complete(getter)
-        signature, ursula_pubkey_sig, ttl, interface_info = msgpack.loads(value.lstrip(b"uaddr-"))
+        signature, ursula_pubkey_sig, hrac, interface_info = dht_value_splitter(value.lstrip(b"uaddr-"), return_remainder=True)
-        port, _interface = msgpack.loads(interface_info)
+        port = msgpack.loads(interface_info)[0]
         assert port in URSULA_PORTS