Merge pull request #142 from jMyles/todo

Closed a few TODOs, opened some others, added some tests for some already closed.

nkms/characters.py

@@ -236,7 +236,7 @@ class Bob(Character):
         self._ursulas = {}
         if alice:
             self.alice = alice
-        self._work_orders = {}
+        self._saved_work_orders = {}
 
     @property
     def alice(self):
@@ -282,19 +282,23 @@ class Bob(Character):
             return TreasureMap(msgpack.loads(packed_node_list))
 
     def generate_work_orders(self, policy_group, *pfrags, num_ursulas=None):
-        # TODO: Perhaps instead of taking a policy_group, it makes more sense for Bob to reconstruct one with the TreasureMap.
+        # TODO: Perhaps instead of taking a policy_group, it makes more sense for Bob to reconstruct one with the TreasureMap.  See #140.
         from nkms.policy.models import WorkOrder  # Prevent circular import
 
-        # existing_work_orders = self._work_orders.get(pfrags, {})  #  TODO: lookup whether we've done this reencryption before - see #137.
-        existing_work_orders = {}
         generated_work_orders = {}
 
         for ursula_dht_key, ursula in self._ursulas.items():
-            if ursula_dht_key in existing_work_orders:
-                continue
-            else:
-                work_order = WorkOrder.constructed_by_bob(policy_group.hrac(), pfrags, ursula_dht_key, self)
-                existing_work_orders[ursula_dht_key] = generated_work_orders[ursula_dht_key] = work_order
+
+            completed_work_orders_for_this_ursula = self._saved_work_orders.setdefault(ursula_dht_key, [])
+
+            pfrags_to_include = []
+            for pfrag in pfrags:
+                if not pfrag in sum([wo.pfrags for wo in completed_work_orders_for_this_ursula], []):  # TODO: This is inane - probably push it down into a WorkOrderHistory concept.
+                    pfrags_to_include.append(pfrag)
+
+            if pfrags_to_include:
+                work_order = WorkOrder.constructed_by_bob(policy_group.hrac(), pfrags_to_include, ursula_dht_key, self)
+                generated_work_orders[ursula_dht_key] = work_order
 
             if num_ursulas is not None:
                 if num_ursulas == len(generated_work_orders):
@@ -302,8 +306,13 @@ class Bob(Character):
 
         return generated_work_orders
 
-    def get_reencrypted_c_frag(self, networky_stuff, work_order):
+    def get_reencrypted_c_frags(self, networky_stuff, work_order):
         cfrags = networky_stuff.reencrypt(work_order)
+        if not len(work_order) == len(cfrags):
+            raise ValueError("Ursula gave back the wrong number of cfrags.  She's up to something.")
+        for counter, pfrag in enumerate(work_order.pfrags):
+            # TODO: Ursula is actually supposed to sign this.  See #141.
+            self._saved_work_orders[work_order.ursula_id].append(work_order)
         return cfrags
 
     def get_ursula(self, ursula_id):
@@ -410,6 +419,7 @@ class Ursula(Character):
         cfrag_byte_stream = b""
 
         for pfrag in work_order.pfrags:
+            # TODO: Sign the result of this.  See #141.
             cfrag_byte_stream += API.ecies_reencrypt(kfrag, pfrag.encrypted_key)
 
         self._work_orders.append(work_order)  # TODO: Put this in Ursula's datastore

nkms/network/node.py

@@ -1,4 +1,7 @@
 from kademlia.node import Node
+
+from nkms.crypto.fragments import CFrag
+from nkms.crypto.utils import RepeatingBytestringSplitter
 from nkms.network.capabilities import ServerCapability
 
 
@@ -28,4 +31,11 @@ class NuCypherNode(Node):
 class NetworkyStuff(object):
 
     def find_ursula(self, id, offer=None):
-        pass
+        pass
+
+    def reencrypt(self, work_order):
+        ursula = self.get_ursula_by_id(work_order.ursula_id)
+        ursula_rest_response = self.send_work_order_payload_to_ursula(work_order, ursula)
+        cfrags = RepeatingBytestringSplitter(CFrag)(ursula_rest_response.content)
+        work_order.complete(cfrags)  # TODO: We'll do verification of Ursula's signature here.  #141
+        return cfrags

nkms/policy/models.py

@@ -308,6 +308,9 @@ class WorkOrder(object):
     def __eq__(self, other):
         return (self.receipt_bytes, self.receipt_signature) == (other.receipt_bytes, other.receipt_signature)
 
+    def __len__(self):
+        return len(self.pfrags)
+
     @classmethod
     def constructed_by_bob(cls, kfrag_hrac, pfrags, ursula_dht_key, bob):
         receipt_bytes = b"wo:" + ursula_dht_key  # TODO: represent the pfrags as bytes and hash them as part of the receipt, ie  + keccak_digest(b"".join(pfrags))  - See #137
@@ -330,3 +333,8 @@ class WorkOrder(object):
         pfrags_as_bytes = [bytes(p) for p in self.pfrags]
         packed_receipt_and_pfrags = msgpack.dumps((self.receipt_bytes, msgpack.dumps(pfrags_as_bytes)))
         return bytes(self.receipt_signature) + self.bob.seal + packed_receipt_and_pfrags
+
+    def complete(self, cfrags):
+        # TODO: Verify that this is in fact complete - right of CFrags and properly signed.
+        # TODO: Mark it complete with datetime.
+        self

tests/characters/test_bob_handles_frags.py

@@ -0,0 +1,88 @@
+from nkms.crypto import api
+from tests.utilities import EVENT_LOOP, MockNetworkyStuff
+
+
+def test_bob_can_follow_treasure_map(enacted_policy_group, ursulas, alice, bob):
+    """
+    Upon receiving a TreasureMap, Bob populates his list of Ursulas with the correct number.
+    """
+    assert len(bob._ursulas) == 0
+    bob.follow_treasure_map(enacted_policy_group.treasure_map)
+    assert len(bob._ursulas) == len(ursulas)
+
+
+def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy_group, alice, bob, ursulas):
+    """
+    Now that Bob has his list of Ursulas, he can issue a WorkOrder to one.  Upon receiving the WorkOrder, Ursula
+    saves it and responds by re-encrypting and giving Bob a cFrag.
+
+    This is a multipart test; it shows proper relations between the Characters Ursula and Bob and also proper
+    interchange between a KFrag, PFrag, and CFrag object in the context of REST-driven proxy re-encryption.
+    """
+
+    # We pick up our story with Bob already having followed the treasure map above, ie:
+    assert len(bob._ursulas) == len(ursulas)
+
+    the_pfrag = enacted_policy_group.pfrag
+
+    # Bob has no saved work orders yet, ever.
+    assert len(bob._saved_work_orders) == 0
+
+    # We'll test against just a single Ursula - here, we make a WorkOrder for just one.
+    work_orders = bob.generate_work_orders(enacted_policy_group, the_pfrag, num_ursulas=1)
+    assert len(work_orders) == 1
+
+    # Bob has saved the WorkOrder, but since he hasn't used it for reencryption yet, it's empty.
+    assert len(bob._saved_work_orders) == 1
+    assert len(list(bob._saved_work_orders.items())[0][1]) == 0
+
+    networky_stuff = MockNetworkyStuff(ursulas)
+
+    ursula_dht_key, work_order = list(work_orders.items())[0]
+
+    # **** RE-ENCRYPTION HAPPENS HERE! ****
+    cfrags = bob.get_reencrypted_c_frags(networky_stuff, work_order)
+    the_cfrag = cfrags[0]  # We only gave one pFrag, so we only got one cFrag.
+
+    # Having received the cFrag, Bob also saved the WorkOrder as complete.
+    assert len(list(bob._saved_work_orders.items())[0][1]) == 1
+
+    # OK, so cool - Bob has his cFrag!  Let's make sure everything went properly.  First, we'll show that it is in fact
+    # the correct cFrag (ie, that Ursula performed reencryption properly).
+    ursula = networky_stuff.get_ursula_by_id(work_order.ursula_id)
+    the_kfrag = ursula.keystore.get_kfrag(work_order.kfrag_hrac)
+    the_correct_cfrag = api.ecies_reencrypt(the_kfrag, the_pfrag.encrypted_key)
+    assert the_cfrag == the_correct_cfrag  # It's the correct cfrag!
+
+    # Now we'll show that Ursula saved the correct WorkOrder.
+    work_orders_from_bob = ursula.work_orders(bob=bob)
+    assert len(work_orders_from_bob) == 1
+    assert work_orders_from_bob[0] == work_order
+
+
+def test_bob_remember_that_he_has_cfrags_for_a_particular_pfrag(enacted_policy_group, alice, bob, ursulas):
+
+    # In our last episode, Bob obtained a cFrag from Ursula.
+    bobs_saved_work_order_map = list(bob._saved_work_orders.items())
+
+    # Bob only has a saved WorkOrder from one Ursula.
+    assert len(bobs_saved_work_order_map) == 1
+
+    id_of_ursula_from_whom_we_already_have_a_cfrag, saved_work_orders = bobs_saved_work_order_map[0]
+
+    # ...and only one WorkOrder from that 1 Ursula.
+    assert len(saved_work_orders) == 1
+
+    # The rest of this test will show that if Bob generates another WorkOrder, it's for a *different* Ursula.
+
+    generated_work_order_map = bob.generate_work_orders(enacted_policy_group, enacted_policy_group.pfrag, num_ursulas=1)
+    id_of_this_new_ursula, new_work_order = list(generated_work_order_map.items())[0]
+
+    # This new Ursula isn't the same one to whom we've already issued a WorkOrder.
+    assert id_of_ursula_from_whom_we_already_have_a_cfrag != id_of_this_new_ursula
+
+    # ...and, although this WorkOrder has the same pfrags as the saved one...
+    new_work_order.pfrags == saved_work_orders[0].pfrags
+
+    # ...it's not the same WorkOrder.
+    assert new_work_order not in saved_work_orders

tests/fixtures.py

@@ -1,4 +1,5 @@
 import datetime
+
 import pytest
 
 from nkms.characters import congregate, Alice, Bob
@@ -27,6 +28,7 @@ def alices_policy_group(alice, bob):
     )
     return policy_group
 
+
 @pytest.fixture(scope="session")
 def enacted_policy_group(alices_policy_group, ursulas):
     # Alice has a policy in mind and knows of enough qualifies Ursulas; she crafts an offer for them.
@@ -36,17 +38,18 @@ def enacted_policy_group(alices_policy_group, ursulas):
 
     networky_stuff = MockNetworkyStuff(ursulas)
     alices_policy_group.find_n_ursulas(networky_stuff, offer)
-    alices_policy_group.enact_policies(networky_stuff)  # REST call happens here.
+    alices_policy_group.enact_policies(networky_stuff)  # REST call happens here, as does population of TreasureMap.
 
     return alices_policy_group
 
+
 @pytest.fixture(scope="session")
-def alice():
+def alice(ursulas):
     ALICE = Alice()
     ALICE.attach_server()
     ALICE.server.listen(8471)
     ALICE.__resource_id = b"some_resource_id"
-    EVENT_LOOP.run_until_complete(ALICE.server.bootstrap([("127.0.0.1", URSULA_PORT)]))
+    EVENT_LOOP.run_until_complete(ALICE.server.bootstrap([("127.0.0.1", u.port) for u in ursulas]))
     return ALICE
 
 
@@ -64,4 +67,10 @@ def bob(alice, ursulas):
 def ursulas():
     URSULAS = make_ursulas(NUMBER_OF_URSULAS_IN_NETWORK, URSULA_PORT)
     yield URSULAS
-    blockchain_client._ursulas_on_blockchain.clear()
+    blockchain_client._ursulas_on_blockchain.clear()
+
+
+@pytest.fixture(scope="session")
+def treasure_map_is_set_on_dht(alice, enacted_policy_group):
+    setter, _, _, _, _ = alice.publish_treasure_map(enacted_policy_group)
+    _set_event = EVENT_LOOP.run_until_complete(setter)

tests/network/test_network_actors.py

@@ -107,6 +107,7 @@ def test_alice_creates_policy_group_with_correct_hrac(alices_policy_group):
         bytes(alice.seal) + bytes(bob.seal) + alice.__resource_id)
 
 
+@pytest.mark.usefixtures("treasure_map_is_set_on_dht")
 def test_alice_sets_treasure_map_on_network(enacted_policy_group, ursulas):
     """
     Having enacted all the policies of a PolicyGroup, Alice creates a TreasureMap and sends it to Ursula via the DHT.
@@ -140,6 +141,7 @@ def test_treasure_map_with_bad_id_does_not_propagate(alices_policy_group, ursula
         ursulas[0].server.storage[digest(illegal_policygroup_id)]
 
 
+@pytest.mark.usefixtures("treasure_map_is_set_on_dht")
 def test_treasure_map_stored_by_ursula_is_the_correct_one_for_bob(alice, bob, ursulas, enacted_policy_group):
     """
     The TreasureMap given by Alice to Ursula is the correct one for Bob; he can decrypt and read it.
@@ -161,6 +163,7 @@ def test_treasure_map_stored_by_ursula_is_the_correct_one_for_bob(alice, bob, ur
     assert verified is True
 
 
+@pytest.mark.usefixtures("treasure_map_is_set_on_dht")
 def test_bob_can_retreive_the_treasure_map_and_decrypt_it(enacted_policy_group, ursulas):
     """
     Above, we showed that the TreasureMap saved on the network is the correct one for Bob.  Here, we show

tests/network/test_network_upgrade.py

@@ -1,7 +1,3 @@
-from nkms.crypto import api
-from tests.utilities import EVENT_LOOP, MockNetworkyStuff
-
-
 def test_alice_enacts_policies_in_policy_group_via_rest(enacted_policy_group):
     """
     Now that Alice has made a PolicyGroup, she can enact its policies, using Ursula's Public Key to encrypt each offer
@@ -10,55 +6,3 @@ def test_alice_enacts_policies_in_policy_group_via_rest(enacted_policy_group):
     ursula = enacted_policy_group.policies[0].ursula
     kfrag_that_was_set = ursula.keystore.get_kfrag(enacted_policy_group.hrac())
     assert bool(kfrag_that_was_set)  # TODO: This can be a more poignant assertion.
-
-
-def test_bob_can_follow_treasure_map(enacted_policy_group, ursulas, alice, bob):
-    """
-    Upon receiving a TreasureMap, Bob populates his list of Ursulas with the correct number.
-    """
-    assert len(bob._ursulas) == 0
-
-    setter, encrypted_treasure_map, packed_encrypted_treasure_map, signature_for_bob, signature_for_ursula = alice.publish_treasure_map(
-        enacted_policy_group)
-    _set_event = EVENT_LOOP.run_until_complete(setter)
-
-    bob.follow_treasure_map(enacted_policy_group.treasure_map)
-    assert len(bob._ursulas) == len(ursulas)
-
-
-def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy_group, alice, bob, ursulas):
-    """
-    Now that Bob has his list of Ursulas, he can issue a WorkOrder to one.  Upon receiving the WorkOrder, Ursula
-    saves it and responds by re-encrypting and giving Bob a cFrag.
-
-    This is a multipart test; it shows proper relations between the Characters Ursula and Bob and also proper
-    interchange between a KFrag, PFrag, and CFrag object in the context of REST-driven proxy re-encryption.
-    """
-
-    # We pick up our story with Bob already having followed the treasure map above, ie:
-    assert len(bob._ursulas) == len(ursulas)
-
-    the_pfrag = enacted_policy_group.pfrag
-
-    # We'll test against just a single Ursula - here, we made a WorkOrder for just one.
-    work_orders = bob.generate_work_orders(enacted_policy_group, the_pfrag, num_ursulas=1)
-    assert len(work_orders) == 1
-
-    networky_stuff = MockNetworkyStuff(ursulas)
-
-    ursula_dht_key, work_order = list(work_orders.items())[0]
-    cfrags = bob.get_reencrypted_c_frag(networky_stuff, work_order)
-
-    the_cfrag = cfrags[0]  # We only gave one pFrag, so we only got one cFrag.
-
-    # Wow, Bob has his cFrag!  Let's make sure everything went properly.  First, we'll show that it is in fact
-    # the correct cFrag (ie, that Ursula performed reencryption properly).
-    ursula = networky_stuff.get_ursula_by_id(work_order.ursula_id)
-    the_kfrag = ursula.keystore.get_kfrag(work_order.kfrag_hrac)
-    the_correct_cfrag = api.ecies_reencrypt(the_kfrag, the_pfrag.encrypted_key)
-    assert the_cfrag == the_correct_cfrag  # It's the correct cfrag!
-
-    # Now we'll show that Ursula saved the correct WorkOrder.
-    work_orders_from_bob = ursula.work_orders(bob=bob)
-    assert len(work_orders_from_bob) == 1
-    assert work_orders_from_bob[0] == work_order

tests/utilities.py

@@ -44,7 +44,7 @@ def make_ursulas(how_many_ursulas: int, ursula_starting_port: int) -> list:
             ursula.server.bootstrap([("127.0.0.1", ursula_starting_port + _c) for _c in range(how_many_ursulas)]))
         ursula.publish_interface_information()
 
-    return URSULAS  # , range(ursula_starting_port, ursula_starting_port + len(URSULAS))
+    return URSULAS
 
 
 class MockPolicyOfferResponse(object):
@@ -81,11 +81,8 @@ class MockNetworkyStuff(NetworkyStuff):
             pytest.fail("No Ursula with ID {}".format(ursula_id))
         return ursula
 
-    def reencrypt(self, work_order):
-        print(work_order)
-        ursula = self.get_ursula_by_id(work_order.ursula_id)
+    def send_work_order_payload_to_ursula(self, work_order, ursula):
         mock_client = TestClient(ursula.rest_app)
         payload = work_order.payload()
-        response = mock_client.post('http://localhost/kFrag/{}/reencrypt'.format(work_order.kfrag_hrac.hex()), payload)
-        cfrags = RepeatingBytestringSplitter(CFrag)(response.content)
-        return cfrags
+        hrac_as_hex = work_order.kfrag_hrac.hex()
+        return mock_client.post('http://localhost/kFrag/{}/reencrypt'.format(hrac_as_hex), payload)