Merge branch 'master' into solc-pipeline

Pipfile

@@ -19,7 +19,7 @@ pyumbral = {git = "https://github.com/nucypher/pyumbral.git"}
 requests = "*"
 hendrix = {git = "https://github.com/hendrix/hendrix", ref = "tags/3.0.0rc1"}
 constantSorrow = {git = "https://github.com/nucypher/constantSorrow.git", ref = "kms-depend"}
-bytestringSplitter = {git = "https://github.com/nucypher/byteStringSplitter.git", ref = "eaa1df2433362190f30bc6e400570f0331980ebb"}
+bytestringSplitter = {git = "https://github.com/nucypher/byteStringSplitter.git", ref = "kms-depend"}
 appdirs = "*"
 eth-tester = "==0.1.0b23"
 py-evm = "*"

examples/alicebob-grant-and-line-by-line-PRE.py

@@ -1,70 +1,77 @@
 # This is an example of Alice setting a Policy on the NuCypher network.
 # In this example, Alice uses n=1, which is almost always a bad idea.  Don't do it.
 
-# WIP w/ hendrix@8227c4abcb37ee6d27528a13ec22d55ee106107f
+# WIP w/ hendrix@3.0.0
 
 import datetime
 import sys
 
-import requests
-
+from examples.sandbox_resources import SandboxNetworkyStuff
 from nkms.characters import Alice, Bob, Ursula
-from nkms.crypto.kits import MessageKit
-from nkms.crypto.powers import SigningPower, EncryptingPower
+from nkms.data_sources import DataSource
 from nkms.network.node import NetworkyStuff
-from umbral import pre
+import maya
 
-ALICE = Alice()
-BOB = Bob()
-URSULA = Ursula.from_rest_url(address="https://localhost", port="3550")
+# This is already running in another process.
+URSULA = Ursula.from_rest_url(NetworkyStuff(), address="localhost", port=3601)
+network_middleware = SandboxNetworkyStuff([URSULA])
 
 
-class SandboxNetworkyStuff(NetworkyStuff):
-    def find_ursula(self, contract=None):
-        ursula = Ursula.as_discovered_on_network(dht_port=None, dht_interface=None,
-                                                 rest_address="https://localhost", rest_port=3550,
-                                                 powers_and_keys={
-                                                    SigningPower: URSULA.stamp.as_umbral_pubkey(),
-                                                    EncryptingPower: URSULA.public_key(EncryptingPower)
-                                                 }
-                                                 )
-        response = requests.post("https://localhost:3550/consider_contract", bytes(contract), verify=False)
-        response.was_accepted = True
-        return ursula, response
+#########
+# Alice #
+#########
 
-    def enact_policy(self, ursula, hrac, payload):
-        response = requests.post('{}:{}/kFrag/{}'.format(ursula.rest_address, ursula.rest_port, hrac.hex()),
-                                 payload, verify=False)
-        # TODO: Something useful here and it's probably ready to go down into NetworkyStuff.
-        return response.status_code == 200
+ALICE = Alice(network_middleware=network_middleware)
 
-
-networky_stuff = SandboxNetworkyStuff()
-
-policy_end_datetime = datetime.datetime.now() + datetime.timedelta(days=5)
+# Here are our Policy details.
+policy_end_datetime = maya.now() + datetime.timedelta(days=5)
+m = 1
 n = 1
-uri = b"secret/files/and/stuff"
+label = b"secret/files/and/stuff"
 
-# Alice gets on the network and discovers Ursula, presumably from the blockchain.
-ALICE.learn_about_nodes(address="https://localhost", port="3550")
+
+# Alice gets on the network and, knowing about at least one Ursula,
+# Is able to discover all Ursulas.
+ALICE.network_bootstrap([("localhost", 3601)])
 
 # Alice grants to Bob.
-
-policy = ALICE.grant(BOB, uri, networky_stuff, m=1, n=n,
+BOB = Bob()
+policy = ALICE.grant(BOB, label, m=m, n=n,
                      expiration=policy_end_datetime)
-policy.publish_treasure_map(networky_stuff, use_dht=False)
-hrac, treasure_map = policy.hrac(), policy.treasure_map
 
-# Bob learns about Ursula, gets the TreasureMap, and follows it.
-BOB.learn_about_nodes(address="https://localhost", port="3550")
-networky_stuff = NetworkyStuff()
-BOB.get_treasure_map(policy, networky_stuff)
-BOB.follow_treasure_map(hrac)
+# Alice puts her public key somewhere for Bob to find later...
+alices_pubkey_saved_for_posterity = bytes(ALICE.stamp)
 
-# Now, Alice and Bob are ready for some throughput.
+# ...and then disappears from the internet.
+del ALICE
+# (this is optional of course - she may wish to remain in order to create
+# new policies in the future.  The point is - she is no longer obligated.
 
+#####################
+# some time passes. #
+# ...               #
+# And now for Bob.  #
+#####################
+
+# Bob wants to join the policy so that he can receive any future
+# data shared on it.
+# He needs a few piece of knowledge to do that.
+BOB.join_policy(label,  # The label - he needs to know what data he's after.
+                alices_pubkey_saved_for_posterity,  # To verify the signature, he'll need Alice's public key.
+                verify_sig=True,  # And yes, he usually wants to verify that signature.
+                # He can also bootstrap himself onto the network more quickly
+                # by providing a list of known nodes at this time.
+                node_list=[("localhost", 3601)]
+                )
+
+# Now that Bob has joined the Policy, let's show how DataSources
+# can share data with the members of this Policy and then how Bob retrieves it.
 finnegans_wake = open(sys.argv[1], 'rb')
 
+# We'll also keep track of some metadata to gauge performance.
+# You can safely ignore from here until...
+################################################################################
+
 start_time = datetime.datetime.now()
 
 for counter, plaintext in enumerate(finnegans_wake):
@@ -78,16 +85,58 @@ for counter, plaintext in enumerate(finnegans_wake):
         print("PREs per second: {}".format(counter / seconds))
         print("********************************")
 
-    ciphertext, capsule = pre.encrypt(ALICE.public_key(EncryptingPower), plaintext)
 
-    message_kit = MessageKit(ciphertext=ciphertext, capsule=capsule,
-                      alice_pubkey=ALICE.public_key(EncryptingPower))
+################################################################################
+# ...here.  OK, pay attention again.
+# Now it's time for...
 
-    work_orders = BOB.generate_work_orders(hrac, capsule)
-    print(plaintext)
-    cfrags = BOB.get_reencrypted_c_frags(networky_stuff, work_orders[bytes(URSULA.stamp)])
+    #####################
+    # Using DataSources #
+    #####################
 
-    capsule.attach_cfrag(cfrags[0])
-    delivered_cleartext = pre.decrypt(capsule, BOB._crypto_power._power_ups[EncryptingPower].keypair._privkey, ciphertext, ALICE.public_key(EncryptingPower))
+    # Now Alice has set a Policy and Bob has joined it.
+    # You're ready to make some DataSources and encrypt for Bob.
+
+    # It may also be helpful to imagine that you have multiple Bobs,
+    # multiple Labels, or both.
+
+    # First we make a DataSource for this policy.
+    data_source = DataSource(policy_pubkey_enc=policy.public_key)
+
+    # Here's how we generate a MessageKit for the Policy.  We also get a signature
+    # here, which can be passed via a side-channel (or posted somewhere public as
+    # testimony) and verified if desired.  In this case, the plaintext is a
+    # single passage from James Joyce's Finnegan's Wake.
+    # The matter of whether encryption makes the passage more or less readable
+    # is left to the reader to determine.
+    message_kit, _signature = data_source.encapsulate_single_message(plaintext)
+
+    # The DataSource will want to be able to be verified by Bob, so it leaves
+    # its Public Key somewhere.
+    data_source_public_key = bytes(data_source.stamp)
+
+    # It can save the MessageKit somewhere (IPFS, etc) and then it too can
+    # choose to disappear (although it may also opt to continue transmitting
+    # as many messages as may be appropriate).
+    del data_source
+
+    ###############
+    # Back to Bob #
+    ###############
+
+    # Bob needs to reconstruct the DataSource.
+    datasource_as_understood_by_bob = DataSource.from_public_keys(
+        policy_public_key=policy.public_key,
+        datasource_public_key=data_source_public_key,
+        label=label
+    )
+
+    # Now Bob can retrieve the original message.  He just needs the MessageKit
+    # and the DataSource which produced it.
+    delivered_cleartext = BOB.retrieve(message_kit=message_kit,
+                                       data_source=datasource_as_understood_by_bob,
+                                       alice_pubkey_sig=alices_pubkey_saved_for_posterity)
+
+    # We show that indeed this is the passage originally encrypted by the DataSource.
     assert plaintext == delivered_cleartext
     print("Retrieved: {}".format(delivered_cleartext))

examples/run_ursula_with_rest_and_dht_but_no_mining.py

@@ -9,7 +9,7 @@ import os
 
 from cryptography.hazmat.primitives.asymmetric import ec
 
-from hendrix.deploy.ssl import HendrixDeployTLS
+from hendrix.deploy.tls import HendrixDeployTLS
 from hendrix.facilities.services import ExistingKeyTLSContextFactory
 from nkms.characters import Ursula
 from OpenSSL.crypto import X509
@@ -19,14 +19,14 @@ from nkms.crypto.api import generate_self_signed_certificate
 
 DB_NAME = "non-mining-proxy-node"
 
-_URSULA = Ursula(dht_port=3501, dht_interface="localhost", db_name=DB_NAME)
-_URSULA.listen()
+_URSULA = Ursula(dht_port=3501, rest_port=3601, ip_address="localhost", db_name=DB_NAME)
+_URSULA.dht_listen()
 
 CURVE = ec.SECP256R1
 cert, private_key = generate_self_signed_certificate(_URSULA.stamp.fingerprint().decode(), CURVE)
 
 deployer = HendrixDeployTLS("start",
-                            {"wsgi":_URSULA.rest_app, "https_port": 3550},
+                            {"wsgi":_URSULA.rest_app, "https_port": _URSULA.rest_port},
                             key=private_key,
                             cert=X509.from_cryptography(cert),
                             context_factory=ExistingKeyTLSContextFactory,

examples/sandbox_resources.py

@@ -0,0 +1,32 @@
+import requests
+from nkms.characters import Ursula
+from nkms.network.node import NetworkyStuff
+from nkms.crypto.powers import SigningPower, EncryptingPower
+
+
+class SandboxNetworkyStuff(NetworkyStuff):
+
+    def __init__(self, ursulas):
+        self.ursulas = ursulas
+
+    def find_ursula(self, contract=None):
+        ursula = Ursula.as_discovered_on_network(dht_port=None,
+                                                 ip_address="localhost",
+                                                 rest_port=3601,
+                                                 powers_and_keys={
+                                                    SigningPower: self.ursulas[0].stamp.as_umbral_pubkey(),
+                                                    EncryptingPower: self.ursulas[0].public_key(EncryptingPower)
+                                                 }
+                                                 )
+        response = requests.post("https://localhost:3601/consider_arrangement", bytes(contract), verify=False)
+        if response.status_code == 200:
+            response.was_accepted = True
+        else:
+            raise RuntimeError("Something went terribly wrong.  What'd you do?!")
+        return ursula, response
+
+    def enact_policy(self, ursula, hrac, payload):
+        endpoint = 'https://{}:{}/kFrag/{}'.format(ursula.ip_address, ursula.rest_port, hrac.hex())
+        response = requests.post(endpoint, payload, verify=False)
+        # TODO: Something useful here and it's probably ready to go down into NetworkyStuff.
+        return response.status_code == 200

nkms/characters.py

@@ -1,17 +1,17 @@
 import asyncio
 from contextlib import suppress
 from logging import getLogger
-
 import msgpack
-import requests
 from collections import OrderedDict
 from kademlia.network import Server
 from kademlia.utils import digest
 from typing import Dict, ClassVar
 from typing import Union, List
+
+from bytestring_splitter import BytestringSplitter
+from nkms.network.node import NetworkyStuff
 from umbral.keys import UmbralPublicKey
 from constant_sorrow import constants, default_constant_splitter
-from bytestring_splitter import RepeatingBytestringSplitter
 
 from nkms.blockchain.eth.actors import PolicyAuthor
 from nkms.config.configs import KMSConfig
@@ -21,7 +21,7 @@ from nkms.crypto.kits import UmbralMessageKit
 from nkms.crypto.powers import CryptoPower, SigningPower, EncryptingPower, DelegatingPower, NoSigningPower
 from nkms.crypto.signature import Signature, signature_splitter, SignatureStamp, StrangerStamp
 from nkms.network import blockchain_client
-from nkms.network.protocols import dht_value_splitter
+from nkms.network.protocols import dht_value_splitter, dht_with_hrac_splitter
 from nkms.network.server import NuCypherDHTServer, NuCypherSeedOnlyDHTServer, ProxyRESTServer
 
 
@@ -37,7 +37,8 @@ class Character(object):
     address = "This is a fake address."  # TODO: #192
 
     def __init__(self, attach_server=True, crypto_power: CryptoPower=None,
-                 crypto_power_ups=None, is_me=True, config: "KMSConfig"=None) -> None:
+                 crypto_power_ups=None, is_me=True, network_middleware=None,
+                 config: "KMSConfig"=None) -> None:
         """
         :param attach_server:  Whether to attach a Server when this Character is
             born.
@@ -72,12 +73,11 @@ class Character(object):
         if crypto_power:
             self._crypto_power = crypto_power
         elif crypto_power_ups:
-            self._crypto_power = CryptoPower(power_ups=crypto_power_ups,
-                                             generate_keys_if_needed=is_me)
+            self._crypto_power = CryptoPower(power_ups=crypto_power_ups)
         else:
-            self._crypto_power = CryptoPower(self._default_crypto_powerups,
-                                             generate_keys_if_needed=is_me)
+            self._crypto_power = CryptoPower(self._default_crypto_powerups)
         if is_me:
+            self.network_middleware = network_middleware or NetworkyStuff()
             try:
                 self._stamp = SignatureStamp(self._crypto_power.power_ups(SigningPower).keypair)
             except NoSigningPower:
@@ -86,6 +86,8 @@ class Character(object):
             if attach_server:
                 self.attach_server()
         else:
+            if network_middleware is not None:
+                raise TypeError("Can't attach network middleware to a Character who isn't me.  What are you even trying to do?")
             self._stamp = StrangerStamp(self._crypto_power.power_ups(SigningPower).keypair)
 
     def __eq__(self, other):
@@ -281,9 +283,33 @@ class Character(object):
         """
         Sends a request to node_url to find out about known nodes.
         """
-        # TODO: Find out about other known nodes, not just this one.  #175
-        node = Ursula.from_rest_url(address, port)
-        self.known_nodes[node.interface_dht_key()] = node
+        response = self.network_middleware.get_nodes_via_rest(address, port)
+        signature, nodes = signature_splitter(response.content, return_remainder=True)
+        # TODO: Although not treasure map-related, this has a whiff of #172.
+        ursula_interface_splitter = dht_value_splitter + BytestringSplitter((bytes, 17))
+        split_nodes = ursula_interface_splitter.repeat(nodes)
+        new_nodes = {}
+        for node_meta in split_nodes:
+            header, sig, pubkey, interface_info = node_meta
+            if not pubkey in self.known_nodes:
+                if sig.verify(keccak_digest(interface_info), pubkey):
+                    address, dht_port, rest_port = msgpack.loads(interface_info)
+                    new_nodes[pubkey] = \
+                        Ursula.as_discovered_on_network(
+                            rest_port=rest_port,
+                            dht_port=dht_port,
+                            ip_address=address.decode("utf-8"),
+                            powers_and_keys=({SigningPower: pubkey})
+                        )
+                else:
+                    message = "Suspicious Activity: Discovered node with bad signature: {}.  Propagated by: {}:{}".format(node_meta, address, port)
+                    self.log.warn(message)
+        return new_nodes
+
+    def network_bootstrap(self, node_list):
+        for node_addr, port in node_list:
+            new_nodes = self.learn_about_nodes(node_addr, port)
+        self.known_nodes.update(new_nodes)
 
 
 class FakePolicyAgent:  # TODO: #192
@@ -298,7 +324,7 @@ class Alice(Character, PolicyAuthor):
         super().__init__(*args, **kwargs)
         PolicyAuthor.__init__(self, self.address, policy_agent=FakePolicyAgent())
 
-    def generate_kfrags(self, bob, m, n) -> List:
+    def generate_kfrags(self, bob, label, m, n) -> List:
         """
         Generates re-encryption key frags ("KFrags") and returns them.
 
@@ -309,27 +335,27 @@ class Alice(Character, PolicyAuthor):
         :param n: Total number of kfrags to generate
         """
         bob_pubkey_enc = bob.public_key(EncryptingPower)
-        return self._crypto_power.power_ups(DelegatingPower).generate_kfrags(bob_pubkey_enc, m, n)
+        return self._crypto_power.power_ups(DelegatingPower).generate_kfrags(bob_pubkey_enc, label, m, n)
 
-    def create_policy(self, bob: "Bob", uri: bytes, m: int, n: int):
+    def create_policy(self, bob: "Bob", label: bytes, m: int, n: int):
         """
         Create a Policy to share uri with bob.
         Generates KFrags and attaches them.
         """
-        kfrags = self.generate_kfrags(bob, m, n)
+        public_key, kfrags = self.generate_kfrags(bob, label, m, n)
         from nkms.policy.models import Policy
         policy = Policy.from_alice(
             alice=self,
+            label=label,
             bob=bob,
             kfrags=kfrags,
-            uri=uri,
+            public_key=public_key,
             m=m,
         )
 
         return policy
 
-    def grant(self, bob, uri, networky_stuff,
-              m=None, n=None, expiration=None, deposit=None):
+    def grant(self, bob, uri, m=None, n=None, expiration=None, deposit=None):
         if not m:
             # TODO: get m from config  #176
             raise NotImplementedError
@@ -342,7 +368,7 @@ class Alice(Character, PolicyAuthor):
         if not deposit:
             default_deposit = None  # TODO: Check default deposit in config.  #176
             if not default_deposit:
-                deposit = networky_stuff.get_competitive_rate()
+                deposit = self.network_middleware.get_competitive_rate()
                 if deposit == NotImplemented:
                     deposit = constants.NON_PAYMENT(b"0000000")
 
@@ -352,11 +378,12 @@ class Alice(Character, PolicyAuthor):
         # by trying differet
         # deposits and expirations on a limited number of Ursulas.
         # Users may decide to inject some market strategies here.
-        found_ursulas = policy.find_ursulas(networky_stuff, deposit,
+        found_ursulas = policy.find_ursulas(self.network_middleware, deposit,
                                             expiration, num_ursulas=n)
         policy.match_kfrags_to_found_ursulas(found_ursulas)
         # REST call happens here, as does population of TreasureMap.
-        policy.enact(networky_stuff)
+        policy.enact(self.network_middleware)
+        policy.publish_treasure_map(self.network_middleware)
 
         return policy  # Now with TreasureMap affixed!
 
@@ -372,33 +399,64 @@ class Bob(Character):
         from nkms.policy.models import WorkOrderHistory  # Need a bigger strategy to avoid circulars.
         self._saved_work_orders = WorkOrderHistory()
 
-    def follow_treasure_map(self, hrac):
-        for ursula_interface_id in self.treasure_maps[hrac]:
-            if ursula_interface_id in self.known_nodes:
-                # If we already know about this Ursula,
-                # we needn't learn about it again.
-                continue
+    def follow_treasure_map(self, hrac, using_dht=False):
 
-            # TODO: perform this part concurrently.
-            value = self.server.get_now(ursula_interface_id)
+        treasure_map = self.treasure_maps[hrac]
+        number_of_known_treasure_ursulas = 0
+        if not using_dht:
+            for ursula_interface_id in treasure_map:
+                pubkey = UmbralPublicKey.from_bytes(ursula_interface_id)
+                if pubkey in self.known_nodes:
+                    number_of_known_treasure_ursulas += 1
 
-            # TODO: Make this much prettier
-            header, signature, ursula_pubkey_sig, _hrac, (
-                port, interface, ttl) = dht_value_splitter(value, msgpack_remainder=True)
+            newly_discovered_nodes = {}
+            nodes_to_check = iter(self.known_nodes.values())
 
-            if header != constants.BYTESTRING_IS_URSULA_IFACE_INFO:
-                raise TypeError("Unknown DHT value.  How did this get on the network?")
+            while number_of_known_treasure_ursulas < treasure_map.m:
+                try:
+                    node_to_check = next(nodes_to_check)
+                except StopIteration:
+                    raise self.NotEnoughUrsulas(
+                        "Unable to follow the TreasureMap; we just don't know enough nodes to ask about this.  Maybe try using the DHT instead.")
 
-            # TODO: If we're going to implement TTL, it will be here.
-            self.known_nodes[ursula_interface_id] = \
-                Ursula.as_discovered_on_network(
-                    dht_port=port,
-                    dht_interface=interface,
-                    powers_and_keys=({SigningPower: ursula_pubkey_sig})
-                )
+                new_nodes = self.learn_about_nodes(node_to_check.ip_address,
+                                                   node_to_check.rest_port)
+                for new_node_pubkey in new_nodes.keys():
+                    if new_node_pubkey in treasure_map:
+                        number_of_known_treasure_ursulas += 1
+                newly_discovered_nodes.update(new_nodes)
 
-    def get_treasure_map(self, policy, networky_stuff, using_dht=False):
-        map_id = policy.treasure_map_dht_key()
+            self.known_nodes.update(newly_discovered_nodes)
+            return newly_discovered_nodes, number_of_known_treasure_ursulas
+        else:
+            for ursula_interface_id in self.treasure_maps[hrac]:
+                pubkey = UmbralPublicKey.from_bytes(ursula_interface_id)
+                if ursula_interface_id in self.known_nodes:
+                    # If we already know about this Ursula,
+                    # we needn't learn about it again.
+                    continue
+
+                if using_dht:
+                    # TODO: perform this part concurrently.
+                    value = self.server.get_now(ursula_interface_id)
+
+                    # TODO: Make this much prettier
+                    header, signature, ursula_pubkey_sig, _hrac, (
+                        port, interface, ttl) = dht_value_splitter(value, msgpack_remainder=True)
+
+                    if header != constants.BYTESTRING_IS_URSULA_IFACE_INFO:
+                        raise TypeError("Unknown DHT value.  How did this get on the network?")
+
+                # TODO: If we're going to implement TTL, it will be here.
+                self.known_nodes[ursula_interface_id] = \
+                    Ursula.as_discovered_on_network(
+                        dht_port=port,
+                        dht_interface=interface,
+                        powers_and_keys=({SigningPower: ursula_pubkey_sig})
+                    )
+
+    def get_treasure_map(self, alice_pubkey_sig, hrac, using_dht=False, verify_sig=True):
+        map_id = keccak_digest(alice_pubkey_sig + hrac)
 
         if using_dht:
             ursula_coro = self.server.get(map_id)
@@ -408,19 +466,26 @@ class Bob(Character):
             if not self.known_nodes:
                 # TODO: Try to find more Ursulas on the blockchain.
                 raise self.NotEnoughUrsulas
-            tmap_message_kit = self.get_treasure_map_from_known_ursulas(networky_stuff, map_id)
+            tmap_message_kit = self.get_treasure_map_from_known_ursulas(self.network_middleware,
+                                                                        map_id)
 
-        verified, packed_node_list = self.verify_from(
-            policy.alice, tmap_message_kit,
-            decrypt=True
-        )
+        if verify_sig:
+            alice = Alice.from_public_keys({SigningPower: alice_pubkey_sig})
+            verified, packed_node_list = self.verify_from(
+                alice, tmap_message_kit,
+                decrypt=True
+            )
+        else:
+            assert False
 
         if not verified:
             return constants.NOT_FROM_ALICE
         else:
             from nkms.policy.models import TreasureMap
-            treasure_map = TreasureMap(msgpack.loads(packed_node_list))
-            self.treasure_maps[policy.hrac()] = treasure_map
+            node_list = msgpack.loads(packed_node_list)
+            m = node_list.pop()
+            treasure_map = TreasureMap(m=m, ursula_interface_ids=node_list)
+            self.treasure_maps[hrac] = treasure_map
             return treasure_map
 
     def get_treasure_map_from_known_ursulas(self, networky_stuff, map_id):
@@ -436,7 +501,7 @@ class Bob(Character):
             if response.status_code == 200 and response.content:
                 # TODO: Make this prettier
                 header, _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = \
-                    dht_value_splitter(response.content, return_remainder=True)
+                    dht_with_hrac_splitter(response.content, return_remainder=True)
                 tmap_messaage_kit = UmbralMessageKit.from_bytes(encrypted_treasure_map)
                 return tmap_messaage_kit
             else:
@@ -459,7 +524,7 @@ class Bob(Character):
                     capsules))
 
         for ursula_dht_key in treasure_map_to_use:
-            ursula = self.known_nodes[ursula_dht_key]
+            ursula = self.known_nodes[UmbralPublicKey.from_bytes(ursula_dht_key)]
 
             capsules_to_include = []
             for capsule in capsules:
@@ -478,8 +543,8 @@ class Bob(Character):
 
         return generated_work_orders
 
-    def get_reencrypted_c_frags(self, networky_stuff, work_order):
-        cfrags = networky_stuff.reencrypt(work_order)
+    def get_reencrypted_c_frags(self, work_order):
+        cfrags = self.network_middleware.reencrypt(work_order)
         if not len(work_order) == len(cfrags):
             raise ValueError("Ursula gave back the wrong number of cfrags.  She's up to something.")
         for counter, capsule in enumerate(work_order.capsules):
@@ -492,20 +557,50 @@ class Bob(Character):
     def get_ursula(self, ursula_id):
         return self._ursulas[ursula_id]
 
+    def join_policy(self, label, alice_pubkey_sig,
+                    using_dht=False, node_list=None, verify_sig=True):
+        hrac = keccak_digest(bytes(alice_pubkey_sig) + bytes(self.stamp) + label)
+        if node_list:
+            self.network_bootstrap(node_list)
+        self.get_treasure_map(alice_pubkey_sig, hrac, using_dht=using_dht, verify_sig=verify_sig)
+        self.follow_treasure_map(hrac, using_dht=using_dht)
+
+    def retrieve(self, message_kit, data_source, alice_pubkey_sig):
+        hrac = keccak_digest(bytes(alice_pubkey_sig) + self.stamp + data_source.label)
+        treasure_map = self.treasure_maps[hrac]
+
+        # First, a quick sanity check to make sure we know about at least m nodes.
+        known_nodes_as_bytes = set([bytes(n) for n in self.known_nodes.keys()])
+        intersection = treasure_map.ids.intersection(known_nodes_as_bytes)
+
+        if len(intersection) < treasure_map.m:
+            raise RuntimeError("Not enough known nodes.  Try following the TreasureMap again.")
+
+        work_orders = self.generate_work_orders(hrac, message_kit.capsule)
+        for node_id in self.treasure_maps[hrac]:
+            node = self.known_nodes[UmbralPublicKey.from_bytes(node_id)]
+            cfrags = self.get_reencrypted_c_frags(work_orders[bytes(node.stamp)])
+            message_kit.capsule.attach_cfrag(cfrags[0])
+        verified, delivered_cleartext = self.verify_from(data_source, message_kit, decrypt=True)
+
+        if verified:
+            return delivered_cleartext
+        else:
+            raise RuntimeError("Not verified - replace this with real message.")
+
 
 class Ursula(Character, ProxyRESTServer):
     _server_class = NuCypherDHTServer
     _alice_class = Alice
     _default_crypto_powerups = [SigningPower, EncryptingPower]
 
-    def __init__(self, dht_port=None, dht_interface=None, dht_ttl=0,
-                 rest_address=None, rest_port=None, db_name=None,
+    def __init__(self, dht_port=None, ip_address=None, dht_ttl=0,
+                 rest_port=None, db_name=None,
                  *args, **kwargs):
         self.dht_port = dht_port
-        self.dht_interface = dht_interface
-        self.dht_ttl = 0
+        self.ip_address = ip_address
         self._work_orders = []
-        ProxyRESTServer.__init__(self, rest_address, rest_port, db_name)
+        ProxyRESTServer.__init__(self, rest_port, db_name)
         super().__init__(*args, **kwargs)
 
     @property
@@ -517,30 +612,28 @@ class Ursula(Character, ProxyRESTServer):
             return self._rest_app
 
     @classmethod
-    def as_discovered_on_network(cls, dht_port, dht_interface,
-                                 rest_address=None, rest_port=None,
-                                 powers_and_keys=()):
+    def as_discovered_on_network(cls, dht_port=None, ip_address=None,
+                                 rest_port=None, powers_and_keys=()):
         # TODO: We also need the encrypting public key here.
         ursula = cls.from_public_keys(powers_and_keys)
         ursula.dht_port = dht_port
-        ursula.dht_interface = dht_interface
-        ursula.rest_address = rest_address
+        ursula.ip_address = ip_address
         ursula.rest_port = rest_port
         return ursula
 
     @classmethod
-    def from_rest_url(cls, address, port):
-        response = requests.get("{}:{}/public_keys".format(address, port), verify=False)  # TODO: TLS-only.
+    def from_rest_url(cls, networky_stuff, address, port):
+        response = networky_stuff.ursula_from_rest_interface(address, port)
         if not response.status_code == 200:
             raise RuntimeError("Got a bad response: {}".format(response))
 
-        key_splitter = RepeatingBytestringSplitter(
+        key_splitter = BytestringSplitter(
             (UmbralPublicKey, PUBLIC_KEY_LENGTH))
-        signing_key, encrypting_key = key_splitter(response.content)
+        signing_key, encrypting_key = key_splitter.repeat(response.content)
 
         stranger_ursula_from_public_keys = cls.from_public_keys(
             {SigningPower: signing_key, EncryptingPower: encrypting_key},
-            rest_address=address,
+            ip_address=address,
             rest_port=port
         )
 
@@ -556,32 +649,25 @@ class Ursula(Character, ProxyRESTServer):
         super().attach_server(ksize, alpha, id, storage)
         self.attach_rest_server(db_name=self.db_name)
 
-    def listen(self):
-        return self.server.listen(self.dht_port, self.dht_interface)
+    def dht_listen(self):
+        return self.server.listen(self.dht_port, self.ip_address)
 
-    def dht_interface_info(self):
-        return self.dht_port, self.dht_interface, self.dht_ttl
+    def interface_information(self):
+        return msgpack.dumps((self.ip_address,
+                       self.dht_port,
+                       self.rest_port))
 
-    def interface_dht_key(self):
-        return bytes(self.stamp)
-        # return self.InterfaceDHTKey(self.stamp, self.interface_hrac())
-
-    def interface_dht_value(self):
-        signature = self.stamp(self.interface_hrac())
-        return (
-                constants.BYTESTRING_IS_URSULA_IFACE_INFO + signature + self.stamp + self.interface_hrac()
-                + msgpack.dumps(self.dht_interface_info())
-        )
-
-    def interface_hrac(self):
-        return keccak_digest(msgpack.dumps(self.dht_interface_info()))
+    def interface_info_with_metadata(self):
+        interface_info = self.interface_information()
+        signature = self.stamp(keccak_digest(interface_info))
+        return constants.BYTESTRING_IS_URSULA_IFACE_INFO + signature + self.stamp + interface_info
 
     def publish_dht_information(self):
         if not self.dht_port and self.dht_interface:
             raise RuntimeError("Must listen before publishing interface information.")
 
-        dht_key = self.interface_dht_key()
-        value = self.interface_dht_value()
+        dht_key = bytes(self.stamp)
+        value = self.interface_info_with_metadata()
         setter = self.server.set(key=dht_key, value=value)
         blockchain_client._ursulas_on_blockchain.append(dht_key)
         loop = asyncio.get_event_loop()

nkms/crypto/kits.py

@@ -3,7 +3,6 @@ from constant_sorrow import constants
 
 
 class CryptoKit:
-    return_remainder_when_splitting = True
     splitter = None
 
     @classmethod
@@ -49,8 +48,14 @@ class MessageKit(CryptoKit):
 
 
 class UmbralMessageKit(MessageKit):
+    return_remainder_when_splitting = True
     splitter = capsule_splitter + key_splitter
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.policy_pubkey = None
+
+    @classmethod
+    def from_bytes(cls, some_bytes):
+        capsule, sender_pubkey_sig, ciphertext = cls.split_bytes(some_bytes)
+        return cls(capsule=capsule, sender_pubkey_sig=sender_pubkey_sig, ciphertext=ciphertext)

nkms/crypto/powers.py

@@ -1,8 +1,10 @@
 import inspect
+from typing import List, Union
 
 from nkms.keystore import keypairs
 from nkms.keystore.keypairs import SigningKeypair, EncryptingKeypair
-from umbral.keys import UmbralPublicKey, UmbralPrivateKey
+from umbral.keys import UmbralPublicKey, UmbralPrivateKey, UmbralKeyingMaterial
+from umbral import pre
 
 
 class PowerUpError(TypeError):
@@ -18,11 +20,10 @@ class NoEncryptingPower(PowerUpError):
 
 
 class CryptoPower(object):
-    def __init__(self, power_ups=None, generate_keys_if_needed=False):
+    def __init__(self, power_ups=None):
         self._power_ups = {}
         # TODO: The keys here will actually be IDs for looking up in a KeyStore.
         self.public_keys = {}
-        self.generate_keys = generate_keys_if_needed
 
         if power_ups is not None:
             for power_up in power_ups:
@@ -36,8 +37,7 @@ class CryptoPower(object):
             power_up_instance = power_up
         elif CryptoPowerUp in inspect.getmro(power_up):
             power_up_class = power_up
-            power_up_instance = power_up(
-                generate_keys_if_needed=self.generate_keys)
+            power_up_instance = power_up()
         else:
             raise TypeError(
                 ("power_up must be a subclass of CryptoPowerUp or an instance "
@@ -75,10 +75,16 @@ class KeyPairBasedPower(CryptoPowerUp):
         elif keypair:
             self.keypair = keypair
         else:
-            # They didn't pass a keypair; we'll make one with the bytes (if any)
-            # they provided.
+            # They didn't pass a keypair; we'll make one with the bytes or
+            # UmbralPublicKey if they provided such a thing.
             if pubkey:
-                key_to_pass_to_keypair = pubkey
+                try:
+                    key_to_pass_to_keypair = pubkey.as_umbral_pubkey()
+                except AttributeError:
+                    try:
+                        key_to_pass_to_keypair = UmbralPublicKey.from_bytes(pubkey)
+                    except TypeError:
+                        key_to_pass_to_keypair = pubkey
             else:
                 # They didn't even pass pubkey_bytes.  We'll generate a keypair.
                 key_to_pass_to_keypair = UmbralPrivateKey.gen_key()
@@ -101,6 +107,13 @@ class KeyPairBasedPower(CryptoPowerUp):
         return self.keypair.pubkey
 
 
+class DerivedKeyBasedPower(CryptoPowerUp):
+    """
+    Rather than rely on an established KeyPair, this type of power
+    derives a key at moments defined by the user.
+    """
+
+
 class SigningPower(KeyPairBasedPower):
     _keypair_class = SigningKeypair
     not_found_error = NoSigningPower
@@ -113,8 +126,23 @@ class EncryptingPower(KeyPairBasedPower):
     provides = ("decrypt",)
 
 
-class DelegatingPower(KeyPairBasedPower):   
-    _keypair_class = EncryptingKeypair
-    not_found_error = PowerUpError
-    provides = ("generate_kfrags",)
+class DelegatingPower(DerivedKeyBasedPower):
 
+    def __init__(self):
+        self.umbral_keying_material = UmbralKeyingMaterial()
+
+    def generate_kfrags(self, bob_pubkey_enc, label, m, n) -> Union[UmbralPublicKey, List]:
+        """
+        Generates re-encryption key frags ("KFrags") and returns them.
+
+        These KFrags can be used by Ursula to re-encrypt a Capsule for Bob so
+        that he can activate the Capsule.
+        :param bob_pubkey_enc: Bob's public key
+        :param m: Minimum number of KFrags needed to rebuild ciphertext
+        :param n: Total number of rekey shares to generate
+        """
+        # TODO: salt?
+
+        __private_key = self.umbral_keying_material.derive_privkey_by_label(label)
+        kfrags = pre.split_rekey(__private_key, bob_pubkey_enc, m, n)
+        return __private_key.get_pubkey(), kfrags

nkms/data_sources.py

@@ -1,15 +1,30 @@
 from nkms.crypto.api import encrypt_and_sign
+from nkms.crypto.signature import SignatureStamp
+from nkms.keystore.keypairs import SigningKeypair
+from constant_sorrow.constants import NO_SIGNING_POWER
+from umbral.keys import UmbralPublicKey
 
 
 class DataSource:
 
-    def __init__(self, policy_pubkey_enc, signer):
-        self._policy_pubkey_enc = policy_pubkey_enc
+    def __init__(self, policy_pubkey_enc, signer=NO_SIGNING_POWER, label=None):
+        self.policy_pubkey = policy_pubkey_enc
+        if signer is NO_SIGNING_POWER:
+            signer = SignatureStamp(SigningKeypair())  # TODO: Generate signing key properly.  #241
         self.stamp = signer
+        self.label = label
 
     def encapsulate_single_message(self, message):
-        message_kit, signature = encrypt_and_sign(self._policy_pubkey_enc,
+        message_kit, signature = encrypt_and_sign(self.policy_pubkey,
                                                   plaintext=message,
                                                   signer=self.stamp)
-        message_kit.policy_pubkey = self._policy_pubkey_enc  # TODO: We can probably do better here.
+        message_kit.policy_pubkey = self.policy_pubkey  # TODO: We can probably do better here.
         return message_kit, signature
+
+    @classmethod
+    def from_public_keys(cls, policy_public_key, datasource_public_key, label):
+        umbral_public_key = UmbralPublicKey.from_bytes(datasource_public_key)
+        return cls(policy_public_key,
+                   signer=SignatureStamp(SigningKeypair(umbral_public_key)),
+                   label=label,
+                   )

nkms/keystore/keypairs.py

@@ -8,7 +8,6 @@ from umbral import pre
 from umbral.config import default_curve
 from nkms.crypto.kits import MessageKit
 from nkms.crypto.signature import Signature
-from typing import List
 
 
 class Keypair(object):
@@ -80,20 +79,6 @@ class EncryptingKeypair(Keypair):
 
         return cleartext
 
-    def generate_kfrags(self, bob_pubkey_enc, m, n) -> List:
-        """
-        Generates re-encryption key frags ("KFrags") and returns them.
-
-        These KFrags can be used by Ursula to re-encrypt a Capsule for Bob so
-        that he can activate the Capsule.
-        :param bob_pubkey_enc: Bob's public key
-        :param m: Minimum number of KFrags needed to rebuild ciphertext
-        :param n: Total number of rekey shares to generate
-        """
-        alice_priv_enc = self._privkey
-        kfrags = pre.split_rekey(alice_priv_enc, bob_pubkey_enc, m, n)
-        return kfrags
-
 
 class SigningKeypair(Keypair):
     """

nkms/keystore/keystore.py

@@ -138,6 +138,9 @@ class KeyStore(object):
         
         policy_arrangement = session.query(PolicyArrangement).filter_by(hrac=hrac_as_hex.encode()).first()
 
+        if policy_arrangement is None:
+            raise NotFound("Can't attach a kfrag to non-existent Arrangement with hrac {}".format(hrac_as_hex))
+
         if policy_arrangement.alice_pubkey_sig.key_data != alice.stamp:
             raise alice.SuspiciousActivity
 

nkms/network/node.py

@@ -1,8 +1,8 @@
 import requests
 from kademlia.node import Node
 
+from bytestring_splitter import BytestringSplitter
 from nkms.crypto.constants import CFRAG_LENGTH
-from bytestring_splitter import RepeatingBytestringSplitter
 from nkms.network.capabilities import ServerCapability
 
 from umbral.fragments import CapsuleFrag
@@ -38,7 +38,7 @@ class NetworkyStuff(object):
 
     def reencrypt(self, work_order):
         ursula_rest_response = self.send_work_order_payload_to_ursula(work_order)
-        cfrags = RepeatingBytestringSplitter((CapsuleFrag, CFRAG_LENGTH))(ursula_rest_response.content)
+        cfrags = BytestringSplitter((CapsuleFrag, CFRAG_LENGTH)).repeat(ursula_rest_response.content)
         work_order.complete(cfrags)  # TODO: We'll do verification of Ursula's signature here.  #141
         return cfrags
 
@@ -46,17 +46,28 @@ class NetworkyStuff(object):
         return NotImplemented
 
     def get_treasure_map_from_node(self, node, map_id):
-        response = requests.get("{}/treasure_map/{}".format(node.rest_url(), map_id.hex()), verify=False)
+        port = node.rest_port
+        address = node.ip_address
+        endpoint = "https://{}:{}/treasure_map/{}".format(address, port, map_id.hex())
+        response = requests.get(endpoint, verify=False)
         return response
 
     def push_treasure_map_to_node(self, node, map_id, map_payload):
-        response = requests.post("{}/treasure_map/{}".format(node.rest_url(), map_id.hex()),
-                      data=map_payload, verify=False)
+        port = node.rest_port
+        address = node.ip_address
+        endpoint = "https://{}:{}/treasure_map/{}".format(address, port, map_id.hex())
+        response = requests.post(endpoint, data=map_payload, verify=False)
         return response
 
     def send_work_order_payload_to_ursula(self, work_order):
         payload = work_order.payload()
-
         hrac_as_hex = work_order.kfrag_hrac.hex()
-        return requests.post('{}/kFrag/{}/reencrypt'.format(work_order.ursula.rest_url(), hrac_as_hex),
+        return requests.post('https://{}/kFrag/{}/reencrypt'.format(work_order.ursula.rest_url(), hrac_as_hex),
                              payload, verify=False)
+
+    def ursula_from_rest_interface(self, address, port):
+        return requests.get("https://{}:{}/public_keys".format(address, port), verify=False)  # TODO: TLS-only.
+
+    def get_nodes_via_rest(self, address, port):
+        response = requests.get("https://{}:{}/list_nodes".format(address, port), verify=False)  # TODO: TLS-only.
+        return response

nkms/network/protocols.py

@@ -11,9 +11,8 @@ from nkms.network.node import NuCypherNode
 from nkms.network.routing import NuCypherRoutingTable
 from umbral.keys import UmbralPublicKey
 
-dht_value_splitter = default_constant_splitter + BytestringSplitter(Signature,
-                                        (UmbralPublicKey, PUBLIC_KEY_LENGTH),
-                                        (bytes, KECCAK_DIGEST_LENGTH))
+dht_value_splitter = default_constant_splitter + BytestringSplitter(Signature, (UmbralPublicKey, PUBLIC_KEY_LENGTH))
+dht_with_hrac_splitter = dht_value_splitter + BytestringSplitter((bytes, KECCAK_DIGEST_LENGTH))
 
 
 class NuCypherHashProtocol(KademliaProtocol):
@@ -21,6 +20,9 @@ class NuCypherHashProtocol(KademliaProtocol):
         super().__init__(sourceNode, storage, ksize, *args, **kwargs)
         self.router = NuCypherRoutingTable(self, ksize, sourceNode)
         self.illegal_keys_seen = []
+        # TODO: This is the wrong way to do this.  See #227.
+        self.treasure_maps = {}
+        self.ursulas = {}
 
     def check_node_for_storage(self, node):
         try:
@@ -75,9 +77,16 @@ class NuCypherHashProtocol(KademliaProtocol):
         self.log.debug("got a store request from %s" % str(sender))
 
         # TODO: Why is this logic here?  This is madness.  See #172.
-        if value.startswith(bytes(constants.BYTESTRING_IS_URSULA_IFACE_INFO)) or value.startswith(
-                bytes(constants.BYTESTRING_IS_TREASURE_MAP)):
-            header, signature, sender_pubkey_sig, hrac, message = dht_value_splitter(
+        if value.startswith(bytes(constants.BYTESTRING_IS_URSULA_IFACE_INFO)):
+            header, signature, sender_pubkey_sig, message = dht_value_splitter(
+                value, return_remainder=True)
+
+            # TODO: TTL?
+            hrac = keccak_digest(message)
+            do_store = self.determine_legality_of_dht_key(signature, sender_pubkey_sig, message,
+                                                          hrac, key, value)
+        elif value.startswith(bytes(constants.BYTESTRING_IS_TREASURE_MAP)):
+            header, signature, sender_pubkey_sig, hrac, message = dht_with_hrac_splitter(
                 value, return_remainder=True)
 
             # TODO: TTL?
@@ -91,6 +100,10 @@ class NuCypherHashProtocol(KademliaProtocol):
         if do_store:
             self.log.info("Storing k/v: {} / {}".format(key, value))
             self.storage[key] = value
+            if value.startswith(bytes(constants.BYTESTRING_IS_URSULA_IFACE_INFO)):
+                self.ursulas[key] = value
+            if value.startswith(bytes(constants.BYTESTRING_IS_TREASURE_MAP)):
+                self.treasure_maps[key] = value
 
         return do_store
 

nkms/network/server.py

@@ -5,7 +5,6 @@ from typing import ClassVar
 
 from apistar import http, Route, App
 from apistar.http import Response
-from bytestring_splitter import BytestringSplitter
 from kademlia.crawling import NodeSpiderCrawl
 from kademlia.network import Server
 from kademlia.utils import digest
@@ -13,12 +12,12 @@ from umbral import pre
 from umbral.fragments import KFrag
 
 from nkms.crypto.kits import UmbralMessageKit
-from nkms.crypto.powers import EncryptingPower, SigningPower, CryptoPower
+from nkms.crypto.powers import EncryptingPower, SigningPower
 from nkms.keystore.threading import ThreadedSession
 from nkms.network.capabilities import SeedOnly, ServerCapability
 from nkms.network.node import NuCypherNode
 from nkms.network.protocols import NuCypherSeedOnlyProtocol, NuCypherHashProtocol, \
-    dht_value_splitter
+    dht_value_splitter, dht_with_hrac_splitter
 from nkms.network.storage import SeedOnlyStorage
 
 
@@ -96,8 +95,7 @@ class NuCypherSeedOnlyDHTServer(NuCypherDHTServer):
 
 class ProxyRESTServer(object):
 
-    def __init__(self, rest_address, rest_port, db_name):
-        self.rest_address = rest_address
+    def __init__(self, rest_port, db_name):
         self.rest_port = rest_port
         self.db_name = db_name
         self._rest_app = None
@@ -117,6 +115,8 @@ class ProxyRESTServer(object):
                   self.reencrypt_via_rest),
             Route('/public_keys', 'GET',
                   self.get_signing_and_encrypting_public_keys),
+            Route('/list_nodes', 'GET',
+                  self.list_all_active_nodes_about_which_we_know),
             Route('/consider_arrangement',
                   'POST',
                   self.consider_arrangement),
@@ -145,14 +145,12 @@ class ProxyRESTServer(object):
         self.db_engine = engine
 
     def rest_url(self):
-        return "{}:{}".format(self.rest_address, self.rest_port)
+        return "{}:{}".format(self.ip_address, self.rest_port)
 
-    # """
+
+    #####################################
     # Actual REST Endpoints and utilities
-    # """
-    # def find_ursulas_by_ids(self, request: http.Request):
-    #
-    #
+    #####################################
 
     def get_signing_and_encrypting_public_keys(self):
         """
@@ -166,15 +164,21 @@ class ProxyRESTServer(object):
 
         return response
 
+    def list_all_active_nodes_about_which_we_know(self):
+        headers = {'Content-Type': 'application/octet-stream'}
+        ursulas_as_bytes = bytes().join(self.server.protocol.ursulas.values())
+        ursulas_as_bytes += self.interface_info_with_metadata()
+        signature = self.stamp(ursulas_as_bytes)
+        return Response(bytes(signature) + ursulas_as_bytes, headers=headers)
+
     def consider_arrangement(self, request: http.Request):
         from nkms.policy.models import Arrangement
-        arrangement, deposit_as_bytes = BytestringSplitter(Arrangement)(request.body, return_remainder=True)
-        arrangement.deposit = deposit_as_bytes
+        arrangement = Arrangement.from_bytes(request.body)
 
         with ThreadedSession(self.db_engine) as session:
-            self.datastore.add_policy_arrangement(
+            new_policyarrangement = self.datastore.add_policy_arrangement(
                 arrangement.expiration.datetime(),
-                arrangement.deposit,
+                bytes(arrangement.deposit),
                 hrac=arrangement.hrac.hex().encode(),
                 alice_pubkey_sig=arrangement.alice.stamp,
                 session=session,
@@ -183,6 +187,7 @@ class ProxyRESTServer(object):
         # to decide if this Arrangement is worth accepting.
 
         headers = {'Content-Type': 'application/octet-stream'}
+        # TODO: Make this a legit response #234.
         return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers)
 
     def set_policy(self, hrac_as_hex, request: http.Request):
@@ -248,25 +253,31 @@ class ProxyRESTServer(object):
     def provide_treasure_map(self, treasure_map_id_as_hex):
         # For now, grab the TreasureMap for the DHT storage.  Soon, no do that.  #TODO!
         treasure_map_id = binascii.unhexlify(treasure_map_id_as_hex)
-        treasure_map_bytes = self.server.storage.get(digest(treasure_map_id))
         headers = {'Content-Type': 'application/octet-stream'}
 
-        return Response(content=treasure_map_bytes, headers=headers)
+        try:
+            treasure_map_bytes = self.server.storage[digest(treasure_map_id)]
+            response = Response(content=treasure_map_bytes, headers=headers)
+        except KeyError:
+            response = Response("No Treasure Map with ID {}".format(treasure_map_id),
+                                status_code=404, headers=headers)
+
+        return response
 
     def receive_treasure_map(self, treasure_map_id_as_hex, request: http.Request):
         # TODO: This function is the epitome of #172.
         treasure_map_id = binascii.unhexlify(treasure_map_id_as_hex)
 
         header, signature_for_ursula, pubkey_sig_alice, hrac, tmap_message_kit = \
-            dht_value_splitter(request.body, return_remainder=True)
+            dht_with_hrac_splitter(request.body, return_remainder=True)
         # TODO: This next line is possibly the worst in the entire codebase at the moment.  #172.
         # Also TODO: TTL?
         do_store = self.server.protocol.determine_legality_of_dht_key(
                     signature_for_ursula, pubkey_sig_alice, tmap_message_kit,
                     hrac, digest(treasure_map_id), request.body)
         if do_store:
-            # TODO: Stop storing things in the protocol storage.  Do this better.
-            # TODO: Propagate to other nodes.
+            # TODO: Stop storing things in the protocol storage.  Do this better.  #227
+            # TODO: Propagate to other nodes.  #235
             self.server.protocol.storage[digest(treasure_map_id)] = request.body
             return # TODO: Proper response here.
         else:

nkms/policy/models.py

@@ -24,7 +24,9 @@ class Arrangement(BlockchainArrangement):
     """
     A Policy must be implemented by arrangements with n Ursulas.  This class tracks the status of that implementation.
     """
-    _EXPECTED_LENGTH = 99
+    _EXPECTED_LENGTH = 106
+    splitter = key_splitter + BytestringSplitter((bytes, KECCAK_DIGEST_LENGTH),
+                                                              (bytes, 27), (bytes, 7))
 
     def __init__(self, alice, hrac, expiration, deposit=None, ursula=None,
                  kfrag=constants.UNKNOWN_KFRAG, alices_signature=None):
@@ -66,10 +68,8 @@ class Arrangement(BlockchainArrangement):
 
     @classmethod
     def from_bytes(cls, arrangement_as_bytes):
-        arrangement_splitter = key_splitter + BytestringSplitter((bytes, KECCAK_DIGEST_LENGTH),
-                                                              (bytes, 27))
-        alice_pubkey_sig, hrac, expiration_bytes, deposit_bytes = arrangement_splitter(
-            arrangement_as_bytes, return_remainder=True)
+        # Still unclear how to arrive at the correct number of bytes to represent a deposit.  See #148.
+        alice_pubkey_sig, hrac, expiration_bytes, deposit_bytes = cls.splitter(arrangement_as_bytes)
         expiration = maya.parse(expiration_bytes.decode())
         alice = Alice.from_public_keys({SigningPower: alice_pubkey_sig})
         return cls(alice=alice, hrac=hrac, expiration=expiration, deposit=int(deposit_bytes))
@@ -114,18 +114,19 @@ class Policy(object):
     """
     _ursula = None
 
-    def __init__(self, alice, bob=None, kfrags=(constants.UNKNOWN_KFRAG,), uri=None, m=None, alices_signature=constants.NOT_SIGNED):
+    def __init__(self, alice, label, bob=None, kfrags=(constants.UNKNOWN_KFRAG,),
+                 public_key=None, m=None, alices_signature=constants.NOT_SIGNED):
 
         """
         :param kfrags:  A list of KFrags to distribute per this Policy.
-        :param uri: The identity of the resource to which Bob is granted access.
+        :param label: The identity of the resource to which Bob is granted access.
         """
         self.alice = alice
+        self.label = label
         self.bob = bob
         self.kfrags = kfrags
-        self.uri = uri
-        self.m = m
-        self.treasure_map = TreasureMap()
+        self.public_key = public_key
+        self.treasure_map = TreasureMap(m=m)
         self._accepted_arrangements = OrderedDict()
 
         self.alices_signature = alices_signature
@@ -155,23 +156,17 @@ class Policy(object):
     @staticmethod
     def from_alice(kfrags,
                    alice,
+                   label,
                    bob,
-                   uri,
+                   public_key,
                    m,
                    ):
         # TODO: What happened to Alice's signature - don't we include it here?
-        policy = Policy(alice, bob, kfrags, uri, m)
+        policy = Policy(alice, label, bob, kfrags, public_key, m)
 
         return policy
 
     def hrac(self):
-        """
-        A convenience method for generating an hrac for this instance.
-        """
-        return self.hrac_for(self.alice, self.bob, self.uri)
-
-    @staticmethod
-    def hrac_for(alice, bob, uri):
         """
         The "hashed resource authentication code".
 
@@ -183,7 +178,7 @@ class Policy(object):
         Alice and Bob have all the information they need to construct this.
         Ursula does not, so we share it with her.
         """
-        return keccak_digest(bytes(alice.stamp) + bytes(bob.stamp) + uri)
+        return keccak_digest(bytes(self.alice.stamp) + bytes(self.bob.stamp) + self.label)
 
     def treasure_map_dht_key(self):
         """
@@ -195,7 +190,7 @@ class Policy(object):
         """
         return keccak_digest(bytes(self.alice.stamp) + self.hrac())
 
-    def publish_treasure_map(self, networky_stuff=None, use_dht=True):
+    def publish_treasure_map(self, networky_stuff=None, use_dht=False):
         if networky_stuff is None and use_dht is False:
             raise ValueError("Can't engage the REST swarm without networky stuff.")
         tmap_message_kit, signature_for_bob = self.alice.encrypt_for(
@@ -210,10 +205,13 @@ class Policy(object):
         map_id = self.treasure_map_dht_key()
 
         if use_dht:
+            # Instead of self.alice, let's say self.author.  See #230.
             setter = self.alice.server.set(map_id, constants.BYTESTRING_IS_TREASURE_MAP + map_payload)
             event_loop = asyncio.get_event_loop()
             event_loop.run_until_complete(setter)
         else:
+            if not self.alice.known_nodes:
+                raise RuntimeError("Alice hasn't learned of any nodes.  Thus, she can't push the TreasureMap.")
             for node in self.alice.known_nodes.values():
                 response = networky_stuff.push_treasure_map_to_node(node, map_id, constants.BYTESTRING_IS_TREASURE_MAP + map_payload)
                 # TODO: Do something here based on success or failure
@@ -271,18 +269,20 @@ class Policy(object):
                 arrangement.publish(kfrag, ursula, result)
                 # TODO: What if there weren't enough Arrangements approved to distribute n kfrags?  We need to raise NotEnoughQualifiedUrsulas.
 
-    def public_key(self):
-        return self.alice.public_key(DelegatingPower)
 
 class TreasureMap(object):
-    def __init__(self, ursula_interface_ids=None):
-        self.ids = ursula_interface_ids or []
+    def __init__(self, m, ursula_interface_ids=None):
+        self.m = m
+        self.ids = set(ursula_interface_ids or set())
 
     def packed_payload(self):
-        return msgpack.dumps([bytes(ursula_id) for ursula_id in self.ids])
+        return msgpack.dumps(self.nodes_as_bytes() + [self.m])
+
+    def nodes_as_bytes(self):
+        return [bytes(ursula_id) for ursula_id in self.ids]
 
     def add_ursula(self, ursula):
-        self.ids.append(ursula.interface_dht_key())
+        self.ids.add(bytes(ursula.stamp))
 
     def __eq__(self, other):
         return self.ids == other.ids
@@ -319,7 +319,7 @@ class WorkOrder(object):
 
     @classmethod
     def construct_by_bob(cls, kfrag_hrac, capsules, ursula, bob):
-        receipt_bytes = b"wo:" + ursula.interface_dht_key()  # TODO: represent the capsules as bytes and hash them as part of the receipt, ie  + keccak_digest(b"".join(capsules))  - See #137
+        receipt_bytes = b"wo:" + ursula.interface_information()  # TODO: represent the capsules as bytes and hash them as part of the receipt, ie  + keccak_digest(b"".join(capsules))  - See #137
         receipt_signature = bob.stamp(receipt_bytes)
         return cls(bob, kfrag_hrac, capsules, receipt_bytes, receipt_signature,
                    ursula)

tests/characters/test_alice_can_grant_and_revoke.py

@@ -7,19 +7,17 @@ from nkms.crypto.api import keccak_digest
 from nkms.crypto.constants import PUBLIC_KEY_LENGTH
 from nkms.crypto.powers import SigningPower, EncryptingPower
 from bytestring_splitter import BytestringSplitter
-from tests.utilities import MockNetworkyStuff
 from umbral.fragments import KFrag
 from umbral.keys import UmbralPublicKey
 import maya
 
 
-def test_grant(alice, bob, ursulas):
-    networky_stuff = MockNetworkyStuff(ursulas)
+def test_grant(alice, bob):
     policy_end_datetime = maya.now() + datetime.timedelta(days=5)
     n = 5
     uri = b"this_is_the_path_to_which_access_is_being_granted"
-    policy = alice.grant(bob, uri, networky_stuff, m=3, n=n,
-                         expiration=policy_end_datetime)
+    policy = alice.grant(bob, uri, m=3, n=n,
+                          expiration=policy_end_datetime)
 
     # The number of policies is equal to the number of Ursulas we're using (n)
     assert len(policy._accepted_arrangements) == n
@@ -42,7 +40,7 @@ def test_grant(alice, bob, ursulas):
     assert found
 
 
-def test_alice_can_get_ursulas_keys_via_rest(alice, ursulas):
+def test_alice_can_get_ursulas_keys_via_rest(ursulas):
     mock_client = TestClient(ursulas[0].rest_app)
     response = mock_client.get('http://localhost/public_keys')
     splitter = BytestringSplitter(

tests/characters/test_bob_handles_frags.py

@@ -5,27 +5,71 @@ from umbral import pre
 from umbral.fragments import KFrag
 
 
-def test_bob_can_follow_treasure_map(enacted_policy, ursulas, alice, bob):
-    """
-    Upon receiving a TreasureMap, Bob populates his list of Ursulas with the correct number.
-    """
+def test_bob_cannot_follow_the_treasure_map_in_isolation(enacted_policy, bob):
 
-    # Simulate Bob finding a TreasureMap on the DHT.
-    # A test to show that Bob can do this can be found in test_network_actors.
+    # Assume for the moment that Bob has already received a TreasureMap, perhaps via a side channel.
     hrac, treasure_map = enacted_policy.hrac(), enacted_policy.treasure_map
     bob.treasure_maps[hrac] = treasure_map
 
     # Bob knows of no Ursulas.
     assert len(bob.known_nodes) == 0
 
-    # ...until he follows the TreasureMap.
-    bob.follow_treasure_map(hrac)
-
-    # Now he knows of all the Ursulas.
-    assert len(bob.known_nodes) == len(treasure_map)
+    # He can't successfully follow the TreasureMap until he learns of a node to ask.
+    with pytest.raises(bob.NotEnoughUrsulas):
+        bob.follow_treasure_map(hrac)
 
 
-def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice, bob, ursulas,
+@pytest.mark.usefixtures("treasure_map_is_set_on_dht")
+def test_bob_can_follow_treasure_map(enacted_policy, ursulas, bob, alice):
+    """
+    Similar to above, but this time, we'll show that if Bob can connect to a single node, he can
+    learn enough to follow the TreasureMap.
+
+    Also, we'll get the TreasureMap from the hrac alone (ie, not via a side channel).
+    """
+    hrac = enacted_policy.hrac()
+
+    # Bob knows of no Ursulas.
+    assert len(bob.known_nodes) == 0
+
+    # Now Bob will ask just a single Ursula to tell him of the nodes about which she's aware.
+    bob.network_bootstrap([("127.0.0.1", ursulas[0].rest_port)])
+
+    # Success!  This Ursula knew about all the nodes on the network!
+    assert len(bob.known_nodes) == len(ursulas)
+
+    # Now, Bob can get the TreasureMap all by himself, and doesn't need a side channel.
+    bob.get_treasure_map(alice.stamp, hrac)
+    newly_discovered, total_known = bob.follow_treasure_map(hrac)
+
+    # He finds that he didn't need to discover any new nodes...
+    assert len(newly_discovered) == 0
+
+    # ...because he already knew of all the Ursulas on the map.
+    assert total_known == len(enacted_policy.treasure_map)
+
+
+def test_bob_can_follow_treasure_map_even_if_he_only_knows_of_one_node(enacted_policy, bob):
+    # Again, let's assume that he received the TreasureMap via a side channel.
+    hrac, treasure_map = enacted_policy.hrac(), enacted_policy.treasure_map
+    bob.treasure_maps[hrac] = treasure_map
+
+    # Now, let's create a scenario in which Bob knows of only one node.
+    k, v = bob.known_nodes.popitem()
+    bob.known_nodes = {k: v}
+    assert len(bob.known_nodes) == 1
+
+    # This time, when he follows the TreasureMap...
+    newly_discovered, total_known = bob.follow_treasure_map(hrac)
+
+    # The newly discovered nodes are all those in the TreasureMap except the one about which he already knew.
+    assert len(newly_discovered) == len(treasure_map) - 1
+
+    # ...and his total known now matches the length of the TreasureMap.
+    assert total_known == len(treasure_map)
+
+
+def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, bob, ursulas,
                                                          capsule_side_channel):
     """
     Now that Bob has his list of Ursulas, he can issue a WorkOrder to one.  Upon receiving the WorkOrder, Ursula
@@ -58,14 +102,10 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice,
     # And the Ursula.
     assert len(bob._saved_work_orders.ursulas) == 1
 
-    networky_stuff = MockNetworkyStuff(ursulas)
     ursula_dht_key, work_order = list(work_orders.items())[0]
 
-    # In the real world, we'll have a full Ursula node here.  But in this case, we need to fake it.
-    work_order.ursula = ursulas[0]
-
     # **** RE-ENCRYPTION HAPPENS HERE! ****
-    cfrags = bob.get_reencrypted_c_frags(networky_stuff, work_order)
+    cfrags = bob.get_reencrypted_c_frags(work_order)
 
     # We only gave one Capsule, so we only got one cFrag.
     assert len(cfrags) == 1
@@ -79,7 +119,13 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice,
 
     # OK, so cool - Bob has his cFrag!  Let's make sure everything went properly.  First, we'll show that it is in fact
     # the correct cFrag (ie, that Ursula performed reencryption properly).
-    ursula = work_order.ursula
+    for u in ursulas:
+        if u.rest_port == work_order.ursula.rest_port:
+            ursula = u
+            break
+    else:
+        raise RuntimeError("Somehow we don't know about this Ursula.  Major malfunction.")
+
     kfrag_bytes = ursula.datastore.get_policy_arrangement(
         work_order.kfrag_hrac.hex().encode()).k_frag
     the_kfrag = KFrag.from_bytes(kfrag_bytes)
@@ -92,7 +138,7 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice,
     assert work_orders_from_bob[0] == work_order
 
 
-def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_policy, alice, bob,
+def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_policy, bob,
                                                                    ursulas, capsule_side_channel):
     # In our last episode, Bob made a WorkOrder for the capsule...
     assert len(bob._saved_work_orders.by_capsule(capsule_side_channel[0].capsule)) == 1
@@ -123,11 +169,7 @@ def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_polic
     assert new_work_order != saved_work_order
 
     # We can get a new CFrag, just like last time.
-    networky_stuff = MockNetworkyStuff(ursulas)
-    # In the real world, we'll have a full Ursula node here.  But in this case, we need to fake it.
-    new_work_order.ursula = ursulas[1]
-
-    cfrags = bob.get_reencrypted_c_frags(networky_stuff, new_work_order)
+    cfrags = bob.get_reencrypted_c_frags(new_work_order)
 
     # Again: one Capsule, one cFrag.
     assert len(cfrags) == 1
@@ -145,23 +187,20 @@ def test_bob_gathers_and_combines(enacted_policy, bob, ursulas, capsule_side_cha
     assert len(bob._saved_work_orders) == 2
 
     # ...but the policy requires us to collect more cfrags.
-    assert len(bob._saved_work_orders) < enacted_policy.m
+    assert len(bob._saved_work_orders) < enacted_policy.treasure_map.m
 
     # Bob can't decrypt yet with just two CFrags.  He needs to gather at least m.
     with pytest.raises(pre.GenericUmbralError):
         bob.decrypt(the_message_kit)
 
-    number_left_to_collect = enacted_policy.m - len(bob._saved_work_orders)
+    number_left_to_collect = enacted_policy.treasure_map.m - len(bob._saved_work_orders)
 
     new_work_orders = bob.generate_work_orders(enacted_policy.hrac(),
                                                the_message_kit.capsule,
                                                num_ursulas=number_left_to_collect)
     _id_of_yet_another_ursula, new_work_order = list(new_work_orders.items())[0]
 
-    networky_stuff = MockNetworkyStuff(ursulas)
-    # In the real world, we'll have a full Ursula node here.  But in this case, we need to fake it.
-    new_work_order.ursula = ursulas[2]
-    cfrags = bob.get_reencrypted_c_frags(networky_stuff, new_work_order)
+    cfrags = bob.get_reencrypted_c_frags(new_work_order)
     the_message_kit.capsule.attach_cfrag(cfrags[0])
 
     # Now.

tests/fixtures.py

@@ -8,6 +8,9 @@ from constant_sorrow import constants
 from sqlalchemy.engine import create_engine
 
 from nkms.characters import Alice, Bob
+from nkms.keystore import keystore
+from nkms.keystore.db import Base
+
 from nkms.crypto.signature import SignatureStamp
 from nkms.data_sources import DataSource
 from nkms.keystore import keystore
@@ -40,7 +43,6 @@ def enacted_policy(idle_policy, ursulas):
     # Alice has a policy in mind and knows of enough qualifies Ursulas; she crafts an offer for them.
     deposit = constants.NON_PAYMENT
     contract_end_datetime = maya.now() + datetime.timedelta(days=5)
-    # contract = Contract(idle_policy.n, deposit, contract_end_datetime)
 
     networky_stuff = MockNetworkyStuff(ursulas)
     found_ursulas = idle_policy.find_ursulas(networky_stuff, deposit, expiration=contract_end_datetime)
@@ -52,18 +54,17 @@ def enacted_policy(idle_policy, ursulas):
 
 @pytest.fixture(scope="module")
 def alice(ursulas):
-    ALICE = Alice()
+    ALICE = Alice(network_middleware=MockNetworkyStuff(ursulas))
     ALICE.server.listen(8471)
     ALICE.__resource_id = b"some_resource_id"
     EVENT_LOOP.run_until_complete(ALICE.server.bootstrap([("127.0.0.1", u.dht_port) for u in ursulas]))
+    ALICE.network_bootstrap([("127.0.0.1", u.rest_port) for u in ursulas])
     return ALICE
 
 
 @pytest.fixture(scope="module")
-def bob():
-    BOB = Bob()
-    BOB.server.listen(8475)
-    EVENT_LOOP.run_until_complete(BOB.server.bootstrap([("127.0.0.1", URSULA_PORT)]))
+def bob(ursulas):
+    BOB = Bob(network_middleware=MockNetworkyStuff(ursulas))
     return BOB
 
 
@@ -79,8 +80,9 @@ def ursulas():
 
 
 @pytest.fixture(scope="module")
-def treasure_map_is_set_on_dht(enacted_policy):
-    enacted_policy.publish_treasure_map()
+def treasure_map_is_set_on_dht(enacted_policy, ursulas):
+    networky_stuff = MockNetworkyStuff(ursulas)
+    enacted_policy.publish_treasure_map(networky_stuff, use_dht=True)
 
 
 @pytest.fixture(scope="module")
@@ -94,7 +96,7 @@ def test_keystore():
 @pytest.fixture(scope="module")
 def capsule_side_channel(enacted_policy):
     signing_keypair = SigningKeypair()
-    data_source = DataSource(policy_pubkey_enc=enacted_policy.public_key(),
+    data_source = DataSource(policy_pubkey_enc=enacted_policy.public_key,
                              signer=SignatureStamp(signing_keypair))
     message_kit, _signature = data_source.encapsulate_single_message(b"Welcome to the flippering.")
     return message_kit, data_source

tests/network/test_network_actors.py

@@ -8,7 +8,7 @@ from kademlia.utils import digest
 from nkms.crypto.api import keccak_digest
 from nkms.crypto.kits import UmbralMessageKit
 from nkms.network import blockchain_client
-from nkms.network.protocols import dht_value_splitter
+from nkms.network.protocols import dht_value_splitter, dht_with_hrac_splitter
 from tests.utilities import MockNetworkyStuff, EVENT_LOOP, URSULA_PORT, NUMBER_OF_URSULAS_IN_NETWORK
 
 
@@ -37,7 +37,7 @@ def test_vladimir_illegal_interface_key_does_not_propagate(ursulas):
     assert ursula.server.protocol.illegal_keys_seen == []
 
     # Vladimir does almost everything right....
-    value = vladimir.interface_dht_value()
+    value = vladimir.interface_info_with_metadata()
 
     # Except he sets an illegal key for his interface.
     illegal_key = b"Not allowed to set arbitrary key for this."
@@ -49,21 +49,34 @@ def test_vladimir_illegal_interface_key_does_not_propagate(ursulas):
     assert digest(illegal_key) in ursula.server.protocol.illegal_keys_seen
 
 
-def test_alice_finds_ursula(alice, ursulas):
+def test_alice_finds_ursula_via_dht(alice, ursulas):
     """
     With the help of any Ursula, Alice can find a specific Ursula.
     """
     ursula_index = 1
     all_ursulas = blockchain_client._ursulas_on_blockchain
     value = alice.server.get_now(all_ursulas[ursula_index])
-    header, _signature, _ursula_pubkey_sig, _hrac, interface_info = dht_value_splitter(value,
+    header, _signature, _ursula_pubkey_sig, interface_info = dht_value_splitter(value,
                                                                                return_remainder=True)
 
     assert header == constants.BYTESTRING_IS_URSULA_IFACE_INFO
-    port = msgpack.loads(interface_info)[0]
+    port = msgpack.loads(interface_info)[1]
     assert port == URSULA_PORT + ursula_index
 
 
+def test_alice_finds_ursula_via_rest(alice, ursulas):
+    networky_stuff = MockNetworkyStuff(ursulas)
+
+    # Imagine alice knows of nobody.
+    alice.known_nodes = {}
+
+    new_nodes = alice.learn_about_nodes(address="https://localhost", port=ursulas[0].rest_port)
+    assert len(new_nodes) == len(ursulas)
+
+    for ursula in ursulas:
+        assert ursula.stamp.as_umbral_pubkey() in new_nodes
+
+
 def test_alice_creates_policy_group_with_correct_hrac(idle_policy):
     """
     Alice creates a PolicyGroup.  It has the proper HRAC, unique per her, Bob, and the uri (resource_id).
@@ -79,7 +92,8 @@ def test_alice_sets_treasure_map_on_network(enacted_policy, ursulas):
     """
     Having enacted all the policies of a PolicyGroup, Alice creates a TreasureMap and sends it to Ursula via the DHT.
     """
-    _, packed_encrypted_treasure_map, _, _ = enacted_policy.publish_treasure_map()
+    networky_stuff = MockNetworkyStuff(ursulas)
+    _, packed_encrypted_treasure_map, _, _ = enacted_policy.publish_treasure_map(networky_stuff=networky_stuff, use_dht=True)
 
     treasure_map_as_set_on_network = ursulas[0].server.storage[
         digest(enacted_policy.treasure_map_dht_key())]
@@ -112,7 +126,7 @@ def test_treasure_map_stored_by_ursula_is_the_correct_one_for_bob(alice, bob, ur
     treasure_map_as_set_on_network = ursulas[0].server.storage[
         digest(enacted_policy.treasure_map_dht_key())]
 
-    header, _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = dht_value_splitter(
+    header, _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = dht_with_hrac_splitter(
         treasure_map_as_set_on_network, return_remainder=True)
 
     assert header == constants.BYTESTRING_IS_TREASURE_MAP
@@ -141,13 +155,13 @@ def test_bob_can_retreive_the_treasure_map_and_decrypt_it(enacted_policy, ursula
 
     # If Bob doesn't know about any Ursulas, he can't find the TreasureMap via the REST swarm:
     with pytest.raises(bob.NotEnoughUrsulas):
-        treasure_map_from_wire = bob.get_treasure_map(enacted_policy, networky_stuff)
+        treasure_map_from_wire = bob.get_treasure_map(enacted_policy.alice.stamp, enacted_policy.hrac())
 
     # Let's imagine he has learned about some - say, from the blockchain.
-    bob.known_nodes = {u.interface_dht_key(): u for u in ursulas}
+    bob.known_nodes = {u.interface_info_with_metadata(): u for u in ursulas}
 
     # Now try.
-    treasure_map_from_wire = bob.get_treasure_map(enacted_policy, networky_stuff)
+    treasure_map_from_wire = bob.get_treasure_map(enacted_policy.alice.stamp, enacted_policy.hrac())
 
     assert enacted_policy.treasure_map == treasure_map_from_wire
 
@@ -159,10 +173,10 @@ def test_treaure_map_is_legit(enacted_policy):
     alice = enacted_policy.alice
     for ursula_interface_id in enacted_policy.treasure_map:
         value = alice.server.get_now(ursula_interface_id)
-        header, signature, ursula_pubkey_sig, hrac, interface_info = dht_value_splitter(value,
+        header, signature, ursula_pubkey_sig, interface_info = dht_value_splitter(value,
                                                                                 return_remainder=True)
         assert header == constants.BYTESTRING_IS_URSULA_IFACE_INFO
-        port = msgpack.loads(interface_info)[0]
+        port = msgpack.loads(interface_info)[1]
         legal_ports = range(NUMBER_OF_URSULAS_IN_NETWORK, NUMBER_OF_URSULAS_IN_NETWORK + URSULA_PORT)
         assert port in legal_ports
 

tests/utilities.py

@@ -26,14 +26,14 @@ def make_ursulas(how_many_ursulas: int, ursula_starting_port: int) -> list:
     URSULAS = []
     for _u in range(how_many_ursulas):
         port = ursula_starting_port + _u
-        _URSULA = Ursula(dht_port=port, dht_interface="127.0.0.1", db_name="test-{}".format(port))
+        _URSULA = Ursula(dht_port=port, ip_address="127.0.0.1", db_name="test-{}".format(port), rest_port=port+100)  # TODO: Make ports unstupid and more clear.
 
         class MockDatastoreThreadPool(object):
             def callInThread(self, f, *args, **kwargs):
                 return f(*args, **kwargs)
 
         _URSULA.datastore_threadpool = MockDatastoreThreadPool()
-        _URSULA.listen()
+        _URSULA.dht_listen()
 
         URSULAS.append(_URSULA)
 
@@ -53,8 +53,9 @@ class MockArrangementResponse(ArrangementResponse):
 
 
 class MockNetworkyStuff(NetworkyStuff):
+
     def __init__(self, ursulas):
-        self._ursulas = {u.interface_dht_key(): u for u in ursulas}
+        self._ursulas = {bytes(u.stamp): u for u in ursulas}
         self.ursulas = iter(ursulas)
 
     def go_live_with_policy(self, ursula, policy_offer):
@@ -67,20 +68,53 @@ class MockNetworkyStuff(NetworkyStuff):
             raise self.NotEnoughQualifiedUrsulas
         mock_client = TestClient(ursula.rest_app)
         response = mock_client.post("http://localhost/consider_arrangement", bytes(arrangement))
+        assert response.status_code == 200
         return ursula, MockArrangementResponse()
 
     def enact_policy(self, ursula, hrac, payload):
-        mock_client = TestClient(ursula.rest_app)
+        rest_app = self._get_rest_app_by_port(ursula.rest_port)
+        mock_client = TestClient(rest_app)
         response = mock_client.post('http://localhost/kFrag/{}'.format(hrac.hex()), payload)
-        return True, ursula.interface_dht_key()
+        assert response.status_code == 200
+        return True, ursula.stamp.as_umbral_pubkey()
+
+    def _get_rest_app_by_port(self, port):
+        for ursula in self._ursulas.values():
+            if ursula.rest_port == port:
+                rest_app = ursula.rest_app
+                break
+        else:
+            raise RuntimeError(
+                "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port))
+        return rest_app
 
     def send_work_order_payload_to_ursula(self, work_order):
-        mock_client = TestClient(work_order.ursula.rest_app)
+        rest_app = self._get_rest_app_by_port(work_order.ursula.rest_port)
+        mock_client = TestClient(rest_app)
         payload = work_order.payload()
         hrac_as_hex = work_order.kfrag_hrac.hex()
         return mock_client.post('http://localhost/kFrag/{}/reencrypt'.format(hrac_as_hex), payload)
 
     def get_treasure_map_from_node(self, node, map_id):
-        mock_client = TestClient(node.rest_app)
+        rest_app = self._get_rest_app_by_port(node.rest_port)
+        mock_client = TestClient(rest_app)
         return mock_client.get("http://localhost/treasure_map/{}".format(map_id.hex()))
 
+    def ursula_from_rest_interface(self, address, port):
+        rest_app = self._get_rest_app_by_port(port)
+        mock_client = TestClient(rest_app)
+        response = mock_client.get("http://localhost/public_keys")
+        return response
+
+    def get_nodes_via_rest(self, address, port):
+        rest_app = self._get_rest_app_by_port(port)
+        mock_client = TestClient(rest_app)
+        response = mock_client.get("http://localhost/list_nodes")
+        return response
+
+    def push_treasure_map_to_node(self, node, map_id, map_payload):
+        rest_app = self._get_rest_app_by_port(node.rest_port)
+        mock_client = TestClient(rest_app)
+        response = mock_client.post("http://localhost/treasure_map/{}".format(map_id.hex()),
+                      data=map_payload, verify=False)
+        return response