Merge pull request #173 from jMyles/policy

Working nodes: Ursula and AliceBob.

entry_points/run_ursula_mining_node.py

@@ -1,25 +0,0 @@
-# This is not an actual mining script.  Don't use this to mine - you won't
-# perform any re-encryptions, and you won't get paid.
-# It might be (but might not be) useful for determining whether you have
-# the proper depedencies and configuration to run an actual mining node.
-
-# WIP w/ hendrix@8227c4abcb37ee6d27528a13ec22d55ee106107f
-
-
-
-from sqlalchemy.engine import create_engine
-
-from nkms.characters import Ursula
-from nkms.keystore import keystore
-from nkms.keystore.db import Base
-
-engine = create_engine('sqlite:///:memory:')
-Base.metadata.create_all(engine)
-ursulas_keystore = keystore.KeyStore(engine)
-_URSULA = Ursula(urulsas_keystore=ursulas_keystore)
-_URSULA.attach_server()
-
-from hendrix.deploy.base import HendrixDeploy
-
-deployer = HendrixDeploy("start", {"wsgi":_URSULA._rest_app, "http_port": 3500})
-deployer.run()

entry_points/run_ursula_with_rest_and_dht_but_no_mining.py

@@ -0,0 +1,22 @@
+# This is not an actual mining script.  Don't use this to mine - you won't
+# perform any re-encryptions, and you won't get paid.
+# It might be (but might not be) useful for determining whether you have
+# the proper depedencies and configuration to run an actual mining node.
+
+# WIP w/ hendrix@83519da900a258d8e27a3b1fedee949414d2de26
+
+import os
+from nkms.characters import Ursula
+DB_NAME = "non-mining-proxy-node"
+
+_URSULA = Ursula(dht_port=3501, dht_interface="localhost", db_name=DB_NAME)
+_URSULA.listen()
+
+from hendrix.deploy.base import HendrixDeploy
+
+deployer = HendrixDeploy("start", {"wsgi":_URSULA.rest_app, "http_port": 3500})
+
+try:
+    deployer.run()
+finally:
+    os.remove(DB_NAME)

examples/alice_sets_policy_with_ursula.py

@@ -1,41 +0,0 @@
-# This is an example of Alice setting a Policy on the NuCypher network.
-# In this example, Alice uses n=1, which is almost always a bad idea.  Don't do it.
-
-# WIP w/ hendrix@8227c4abcb37ee6d27528a13ec22d55ee106107f
-
-import datetime
-
-import requests
-
-from nkms.characters import Alice, Bob, Ursula
-from nkms.network.node import NetworkyStuff
-
-ALICE = Alice()
-BOB = Bob()
-URSULA = Ursula.from_rest_url("http://localhost:3500/public_keys")
-
-ALICE.learn_about_actor(URSULA)
-
-
-class SandboxNetworkyStuff(NetworkyStuff):
-    def find_ursula(self, contract=None):
-        ursula = Ursula.as_discovered_on_network(dhr_port=None, dht_interface=None, pubkey_sig_bytes=bytes(URSULA.stamp),
-                                                 rest_address="localhost", rest_port=3500)
-        response = requests.post("http://localhost:3500/consider_contract", bytes(contract))
-        response.was_accepted = True
-        return ursula, response
-
-    def enact_policy(self, ursula, hrac, payload):
-        response = requests.post('http://{}:{}/kFrag/{}'.format(ursula.rest_address, ursula.rest_port, hrac.hex()),
-                                 payload)
-        # TODO: Something useful here and it's probably ready to go down into NetworkyStuff.
-        return response.status_code == 200
-
-
-networky_stuff = SandboxNetworkyStuff()
-
-policy_end_datetime = datetime.datetime.now() + datetime.timedelta(days=5)
-n = 1
-uri = b"secret/files/and/stuff"
-policy = ALICE.grant(BOB, uri, networky_stuff, m=1, n=n,
-                     expiration=policy_end_datetime)

examples/alicebob-grant-and-line-by-line-PRE.py

@@ -0,0 +1,93 @@
+# This is an example of Alice setting a Policy on the NuCypher network.
+# In this example, Alice uses n=1, which is almost always a bad idea.  Don't do it.
+
+# WIP w/ hendrix@8227c4abcb37ee6d27528a13ec22d55ee106107f
+
+import datetime
+import sys
+
+import requests
+
+from nkms.characters import Alice, Bob, Ursula
+from nkms.crypto.kits import MessageKit
+from nkms.crypto.powers import SigningPower, EncryptingPower
+from nkms.network.node import NetworkyStuff
+from umbral import pre
+
+ALICE = Alice()
+BOB = Bob()
+URSULA = Ursula.from_rest_url(address="http://localhost", port="3500")
+
+
+class SandboxNetworkyStuff(NetworkyStuff):
+    def find_ursula(self, contract=None):
+        ursula = Ursula.as_discovered_on_network(dht_port=None, dht_interface=None,
+                                                 rest_address="localhost", rest_port=3500,
+                                                 powers_and_keys={
+                                                    SigningPower: URSULA.stamp.as_umbral_pubkey(),
+                                                    EncryptingPower: URSULA.public_key(EncryptingPower)
+                                                 }
+                                                 )
+        response = requests.post("http://localhost:3500/consider_contract", bytes(contract))
+        response.was_accepted = True
+        return ursula, response
+
+    def enact_policy(self, ursula, hrac, payload):
+        response = requests.post('http://{}:{}/kFrag/{}'.format(ursula.rest_address, ursula.rest_port, hrac.hex()),
+                                 payload)
+        # TODO: Something useful here and it's probably ready to go down into NetworkyStuff.
+        return response.status_code == 200
+
+
+networky_stuff = SandboxNetworkyStuff()
+
+policy_end_datetime = datetime.datetime.now() + datetime.timedelta(days=5)
+n = 1
+uri = b"secret/files/and/stuff"
+
+# Alice gets on the network and discovers Ursula, presumably from the blockchain.
+ALICE.learn_about_nodes(address="http://localhost", port="3500")
+
+# Alice grants to Bob.
+
+policy = ALICE.grant(BOB, uri, networky_stuff, m=1, n=n,
+                     expiration=policy_end_datetime)
+policy.publish_treasure_map(networky_stuff, use_dht=False)
+hrac, treasure_map = policy.hrac(), policy.treasure_map
+
+# Bob learns about Ursula, gets the TreasureMap, and follows it.
+BOB.learn_about_nodes(address="http://localhost", port="3500")
+networky_stuff = NetworkyStuff()
+BOB.get_treasure_map(policy, networky_stuff)
+BOB.follow_treasure_map(hrac)
+
+# Now, Alice and Bob are ready for some throughput.
+
+finnegans_wake = open(sys.argv[1], 'rb')
+
+start_time = datetime.datetime.now()
+
+for counter, plaintext in enumerate(finnegans_wake):
+    if counter % 20 == 0:
+        now_time = datetime.datetime.now()
+        time_delta = now_time - start_time
+        seconds = time_delta.total_seconds()
+        print("********************************")
+        print("Performed {} PREs".format(counter))
+        print("Elapsed: {}".format(time_delta.total_seconds()))
+        print("PREs per second: {}".format(counter / seconds))
+        print("********************************")
+
+    ciphertext, capsule = pre.encrypt(ALICE.public_key(EncryptingPower), plaintext)
+
+    message_kit = MessageKit(ciphertext=ciphertext, capsule=capsule,
+                      alice_pubkey=ALICE.public_key(EncryptingPower))
+
+    work_orders = BOB.generate_work_orders(hrac, capsule)
+    print(plaintext)
+    cfrags = BOB.get_reencrypted_c_frags(networky_stuff, work_orders[bytes(URSULA.stamp)])
+
+    capsule.attach_cfrag(cfrags[0])
+    delivered_cleartext = pre.decrypt(capsule, BOB._crypto_power._power_ups[EncryptingPower].keypair._privkey, ciphertext, ALICE.public_key(EncryptingPower))
+    assert plaintext == delivered_cleartext
+    print("Retrieved: {}".format(delivered_cleartext))

examples/set_dht_value_without_listening.py

@@ -1,20 +0,0 @@
-import asyncio
-import logging
-
-from kademlia.network import Server
-from nkms.network.server import NuCypherSeedOnlyDHTServer
-
-key = "llamas"
-value = "tons_of_things_keyed_llamas"
-
-logging.basicConfig(level=logging.DEBUG)
-loop = asyncio.get_event_loop()
-loop.set_debug(True)
-
-server = NuCypherSeedOnlyDHTServer()
-server.listen(8469)
-loop.run_until_complete(server.bootstrap([("127.0.0.1", 8468)]))
-set = server.set(key, value)
-loop.run_until_complete(set)
-server.stop()
-loop.close()

nkms/characters.py

@@ -3,11 +3,13 @@ from logging import getLogger
 
 import msgpack
 import requests
-from apistar.core import Route
-from apistar.frameworks.wsgi import WSGIApp as App
+
+from typing import Dict
+
 from kademlia.network import Server
 from kademlia.utils import digest
 from typing import Union, List
+from collections import OrderedDict
 
 from nkms.crypto.api import secure_random, keccak_digest
 from nkms.crypto.constants import NOT_SIGNED, NO_DECRYPTION_PERFORMED
@@ -56,6 +58,7 @@ class Character(object):
             Character, but there are scenarios in which its imaginable to be
             represented by zero Characters or by more than one Character.
         """
+        self.known_nodes = {}
         self.log = getLogger("characters")
         if crypto_power and crypto_power_ups:
             raise ValueError("Pass crypto_power or crypto_power_ups (or neither), but not both.")
@@ -83,20 +86,22 @@ class Character(object):
     def __hash__(self):
         return int.from_bytes(self.stamp, byteorder="big")
 
-    class NotFound(KeyError):
-        """raised when we try to interact with an actor of whom we haven't \
-           learned yet."""
+    class NotEnoughUrsulas(RuntimeError):
+        """
+        All Characters depend on knowing about enough Ursulas to perform their role.
+        This exception is raised when a piece of logic can't proceed without more Ursulas.
+        """
 
     class SuspiciousActivity(RuntimeError):
         """raised when an action appears to amount to malicious conduct."""
 
     @classmethod
-    def from_public_keys(cls, *powers_and_keys):
+    def from_public_keys(cls, powers_and_keys: Dict, *args, **kwargs):
         """
         Sometimes we discover a Character and, at the same moment, learn one or
-        more of their public keys. Here, we take a collection of tuples
+        more of their public keys. Here, we take a Dict
         (powers_and_key_bytes) in the following format:
-        (CryptoPowerUp class, public_key_bytes)
+        {CryptoPowerUp class: public_key_bytes}
 
         Each item in the collection will have the CryptoPowerUp instantiated
         with the public_key_bytes, and the resulting CryptoPowerUp instance
@@ -104,18 +109,21 @@ class Character(object):
         """
         crypto_power = CryptoPower()
 
-        for power_up, public_key in powers_and_keys:
+        for power_up, public_key in powers_and_keys.items():
             try:
                 umbral_key = UmbralPublicKey(public_key)
             except TypeError:
                 umbral_key = public_key
 
+
             crypto_power.consume_power_up(power_up(pubkey=umbral_key))
 
-        return cls(is_me=False, crypto_power=crypto_power)
+        return cls(is_me=False, crypto_power=crypto_power, *args, **kwargs)
 
     def attach_server(self, ksize=20, alpha=3, id=None,
                       storage=None, *args, **kwargs) -> None:
+        if self._server:
+            raise RuntimeError("Attaching the server twice is almost certainly a bad idea.")
         self._server = self._server_class(
             ksize, alpha, id, storage, *args, **kwargs)
 
@@ -166,13 +174,15 @@ class Character(object):
                 # Encrypt first, sign second.
                 ciphertext, capsule = pre.encrypt(recipient_pubkey_enc, plaintext)
                 signature = self.stamp(ciphertext)
+            alice_pubkey = self.public_key(SigningPower)
         else:
             # Don't sign.
             signature = NOT_SIGNED
             ciphertext, capsule = pre.encrypt(recipient_pubkey_enc, plaintext)
+            alice_pubkey = None
+
+        message_kit = MessageKit(ciphertext=ciphertext, capsule=capsule, alice_pubkey=alice_pubkey)
 
-        message_kit = MessageKit(ciphertext=ciphertext, capsule=capsule)
-        message_kit.alice_pubkey = self.public_key(SigningPower)
         return message_kit, signature
 
     def verify_from(self,
@@ -253,6 +263,14 @@ class Character(object):
         power_up = self._crypto_power.power_ups(power_up_class)
         return power_up.public_key()
 
+    def learn_about_nodes(self, address, port):
+        """
+        Sends a request to node_url to find out about known nodes.
+        """
+        # TODO: Find out about other known nodes, not just this one.
+        node = Ursula.from_rest_url(address, port)
+        self.known_nodes[node.interface_dht_key()] = node
+
 
 class Alice(Character):
     _server_class = NuCypherSeedOnlyDHTServer
@@ -333,7 +351,6 @@ class Bob(Character):
 
     def __init__(self, alice=None, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._ursulas = {}
         self.treasure_maps = {}
         if alice:
             self.alice = alice
@@ -353,6 +370,11 @@ class Bob(Character):
 
     def follow_treasure_map(self, hrac):
         for ursula_interface_id in self.treasure_maps[hrac]:
+            if ursula_interface_id in self.known_nodes:
+                # If we already know about this Ursula,
+                # we needn't learn about it again.
+                continue
+
             # TODO: perform this part concurrently.
             value = self.server.get_now(ursula_interface_id)
 
@@ -363,31 +385,29 @@ class Bob(Character):
                 raise TypeError("Unknown DHT value.  How did this get on the network?")
 
             # TODO: If we're going to implement TTL, it will be here.
-            self._ursulas[ursula_interface_id] =\
+            self.known_nodes[ursula_interface_id] =\
                     Ursula.as_discovered_on_network(
                             dht_port=port,
                             dht_interface=interface,
-                            pubkey_sig_bytes=ursula_pubkey_sig
+                            powers_and_keys=({SigningPower: ursula_pubkey_sig})
                     )
 
-    def get_treasure_map(self, policy_group):
+    def get_treasure_map(self, policy, networky_stuff, using_dht=False):
 
-        dht_key = policy_group.treasure_map_dht_key()
+        map_id = policy.treasure_map_dht_key()
 
-        ursula_coro = self.server.get(dht_key)
-        event_loop = asyncio.get_event_loop()
-        packed_encrypted_treasure_map = event_loop.run_until_complete(ursula_coro)
-        
-        # TODO: Make this prettier
-        header, _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map =\
-        dht_value_splitter(packed_encrypted_treasure_map, return_remainder=True)
-        tmap_messaage_kit = MessageKit.from_bytes(encrypted_treasure_map)
-
-        if header != BYTESTRING_IS_TREASURE_MAP:
-            raise TypeError("Unknown DHT value.  How did this get on the network?")
+        if using_dht:
+            ursula_coro = self.server.get(map_id)
+            event_loop = asyncio.get_event_loop()
+            packed_encrypted_treasure_map = event_loop.run_until_complete(ursula_coro)
+        else:
+            if not self.known_nodes:
+                # TODO: Try to find more Ursulas on the blockchain.
+                raise self.NotEnoughUrsulas
+            tmap_message_kit = self.get_treasure_map_from_known_ursulas(networky_stuff, map_id)
 
         verified, packed_node_list = self.verify_from(
-            self.alice, tmap_messaage_kit,
+            self.alice, tmap_message_kit,
             signature_is_on_cleartext=True, decrypt=True
         )
 
@@ -395,10 +415,28 @@ class Bob(Character):
             return NOT_FROM_ALICE
         else:
             from nkms.policy.models import TreasureMap
-            self.treasure_maps[policy_group.hrac] = TreasureMap(
-                msgpack.loads(packed_node_list)
-            )
-            return self.treasure_maps[policy_group.hrac]
+            treasure_map = TreasureMap(msgpack.loads(packed_node_list))
+            self.treasure_maps[policy.hrac()] = treasure_map
+            return treasure_map
+
+    def get_treasure_map_from_known_ursulas(self, networky_stuff, map_id):
+        """
+        Iterate through swarm, asking for the TreasureMap.
+        Return the first one who has it.
+        TODO: What if a node gives a bunk TreasureMap?
+        """
+        from nkms.network.protocols import dht_value_splitter
+        for node in self.known_nodes.values():
+            response = networky_stuff.get_treasure_map_from_node(node, map_id)
+
+            if response.status_code == 200 and response.content:
+                # TODO: Make this prettier
+                header, _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = \
+                    dht_value_splitter(response.content, return_remainder=True)
+                tmap_messaage_kit = MessageKit.from_bytes(encrypted_treasure_map)
+                return tmap_messaage_kit
+            else:
+                assert False
 
     def generate_work_orders(self, kfrag_hrac, *capsules, num_ursulas=None):
         from nkms.policy.models import WorkOrder  # Prevent circular import
@@ -409,7 +447,7 @@ class Bob(Character):
             raise KeyError(
                 "Bob doesn't have a TreasureMap matching the hrac {}".format(kfrag_hrac))
 
-        generated_work_orders = {}
+        generated_work_orders = OrderedDict()
 
         if not treasure_map_to_use:
             raise ValueError(
@@ -417,7 +455,7 @@ class Bob(Character):
                     capsules))
 
         for ursula_dht_key in treasure_map_to_use:
-            ursula = self._ursulas[ursula_dht_key]
+            ursula = self.known_nodes[ursula_dht_key]
 
             capsules_to_include = []
             for capsule in capsules:
@@ -426,9 +464,9 @@ class Bob(Character):
 
             if capsules_to_include:
                 work_order = WorkOrder.construct_by_bob(
-                    kfrag_hrac, capsules_to_include, ursula_dht_key, self)
+                    kfrag_hrac, capsules_to_include, ursula, self)
                 generated_work_orders[ursula_dht_key] = work_order
-                self._saved_work_orders[work_order.ursula_id][capsule] = work_order
+                self._saved_work_orders[ursula_dht_key][capsule] = work_order
 
             if num_ursulas is not None:
                 if num_ursulas == len(generated_work_orders):
@@ -443,7 +481,7 @@ class Bob(Character):
         for counter, capsule in enumerate(work_order.capsules):
             # TODO: Ursula is actually supposed to sign this.  See #141.
             # TODO: Maybe just update the work order here instead of setting it anew.
-            work_orders_by_ursula = self._saved_work_orders[work_order.ursula_id]
+            work_orders_by_ursula = self._saved_work_orders[bytes(work_order.ursula.stamp)]
             work_orders_by_ursula[capsule] = work_order
         return cfrags
 
@@ -456,18 +494,15 @@ class Ursula(Character, ProxyRESTServer):
     _alice_class = Alice
     _default_crypto_powerups = [SigningPower, EncryptingPower]
 
-    dht_port = None
-    dht_interface = None
-    dht_ttl = 0
-    rest_address = None
-    rest_port = None
-
-    def __init__(self, urulsas_keystore=None, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.keystore = urulsas_keystore
-
-        self._rest_app = None
+    def __init__(self, dht_port=None, dht_interface=None, dht_ttl=0,
+                 rest_address=None, rest_port=None, db_name=None,
+                 *args, **kwargs):
+        self.dht_port = dht_port
+        self.dht_interface = dht_interface
+        self.dht_ttl = 0
         self._work_orders = []
+        ProxyRESTServer.__init__(self, rest_address, rest_port, db_name)
+        super().__init__(*args, **kwargs)
 
     @property
     def rest_app(self):
@@ -489,8 +524,8 @@ class Ursula(Character, ProxyRESTServer):
         return ursula
 
     @classmethod
-    def from_rest_url(cls, url):
-        response = requests.get(url)
+    def from_rest_url(cls, address, port):
+        response = requests.get("{}:{}/public_keys".format(address, port))  # TODO: TLS-only.
         if not response.status_code == 200:
             raise RuntimeError("Got a bad response: {}".format(response))
         signing_key_bytes, encrypting_key_bytes = \
@@ -508,53 +543,17 @@ class Ursula(Character, ProxyRESTServer):
             id = digest(secure_random(32))
 
         super().attach_server(ksize, alpha, id, storage)
+        self.attach_rest_server(db_name=self.db_name)
 
-        routes = [
-            Route('/kFrag/{hrac_as_hex}',
-                  'POST',
-                  self.set_policy),
-            Route('/kFrag/{hrac_as_hex}/reencrypt',
-                  'POST',
-                  self.reencrypt_via_rest),
-            Route('/public_keys', 'GET',
-                  self.get_signing_and_encrypting_public_keys),
-            Route('/consider_contract',
-                  'POST',
-                  self.consider_contract),
-        ]
-
-        self._rest_app = App(routes=routes)
-
-    def listen(self, port, interface):
-        self.dht_port = port
-        self.dht_interface = interface
-        return self.server.listen(port, interface)
+    def listen(self):
+        return self.server.listen(self.dht_port, self.dht_interface)
 
     def dht_interface_info(self):
         return self.dht_port, self.dht_interface, self.dht_ttl
 
-    class InterfaceDHTKey:
-        def __init__(self, stamp, interface_hrac):
-            self.pubkey_sig_bytes = bytes(stamp)
-            self.interface_hrac = interface_hrac
-
-        def __bytes__(self):
-            return keccak_digest(self.pubkey_sig_bytes + self.interface_hrac)
-
-        def __add__(self, other):
-            return bytes(self) + other
-
-        def __radd__(self, other):
-            return other + bytes(self)
-
-        def __hash__(self):
-            return int.from_bytes(self, byteorder="big")
-
-        def __eq__(self, other):
-            return bytes(self) == bytes(other)
-
     def interface_dht_key(self):
-        return self.InterfaceDHTKey(self.stamp, self.interface_hrac())
+        return bytes(self.stamp)
+        # return self.InterfaceDHTKey(self.stamp, self.interface_hrac())
 
     def interface_dht_value(self):
         signature = self.stamp(self.interface_hrac())
@@ -606,6 +605,9 @@ class SignatureStamp(object):
     def __bytes__(self):
         return bytes(self.character.public_key(SigningPower))
 
+    def __hash__(self):
+        return int.from_bytes(self, byteorder="big")
+
     def __eq__(self, other):
         return other == bytes(self)
 

nkms/crypto/powers.py

@@ -64,8 +64,9 @@ class CryptoPowerUp(object):
 class KeyPairBasedPower(CryptoPowerUp):
     _keypair_class = keypairs.Keypair
 
-    def __init__(self, keypair: keypairs.Keypair = None,
+    def __init__(self,
                  pubkey: UmbralPublicKey = None,
+                 keypair: keypairs.Keypair = None,
                  generate_keys_if_needed=True) -> None:
         if keypair and pubkey:
             raise ValueError(

nkms/keystore/db/models.py

@@ -1,6 +1,8 @@
 import sha3
 
 from datetime import datetime
+
+from nkms.crypto.utils import fingerprint_from_key
 from nkms.keystore.db import Base
 from sqlalchemy.orm import relationship
 
@@ -23,17 +25,22 @@ class Key(Base):
         self.key_data = key_data
         self.is_signing = is_signing
 
+    @classmethod
+    def from_umbral_key(cls, umbral_key, is_signing):
+        fingerprint = fingerprint_from_key(umbral_key)
+        key_data = bytes(umbral_key)
+        return cls(fingerprint, key_data, is_signing)
+
 
 class PolicyContract(Base):
     __tablename__ = 'policycontracts'
 
-    id = Column(Integer, primary_key=True)
+    hrac = Column(LargeBinary, unique=True, primary_key=True)
     expiration = Column(DateTime)
     deposit = Column(LargeBinary)
-    hrac = Column(LargeBinary, unique=True)
     k_frag = Column(LargeBinary, unique=True, nullable=True)
     alice_pubkey_sig_id = Column(Integer, ForeignKey('keys.id'))
-    alice_pubkey_sig = relationship(Key, backref="policies")
+    alice_pubkey_sig = relationship(Key, backref="policies", lazy='joined')
     # alice_pubkey_enc_id = Column(Integer, ForeignKey('keys.id'))
     # bob_pubkey_sig_id = Column(Integer, ForeignKey('keys.id'))
     # TODO: Maybe this will be two signatures - one for the offer, one for the KFrag.

nkms/keystore/keystore.py

@@ -1,15 +1,15 @@
 from typing import Union
 
-from sqlalchemy.orm import sessionmaker
-
 from nkms.crypto.constants import KFRAG_LENGTH
 from nkms.crypto.signature import Signature
 from nkms.crypto.utils import BytestringSplitter
 from nkms.keystore.db.models import Key, PolicyContract, Workorder
+from nkms.keystore.threading import ThreadedSession
 from umbral.fragments import KFrag
 from umbral.keys import UmbralPublicKey
 from . import keypairs
 from nkms.crypto.utils import fingerprint_from_key
+from sqlalchemy.orm import sessionmaker
 
 
 class NotFound(Exception):
@@ -31,26 +31,30 @@ class KeyStore(object):
 
         :param sqlalchemy_engine: SQLAlchemy engine object to create session
         """
+        self.engine = sqlalchemy_engine
         Session = sessionmaker(bind=sqlalchemy_engine)
-        self.session = Session()
 
-    def add_key(self, key, is_signing=True) -> Key:
+        # This will probably be on the reactor thread for most production configs.
+        # Best to treat like hot lava.
+        self._session_on_init_thread = Session()
+
+    def add_key(self, key, is_signing=True, session=None) -> Key:
         """
         :param key: Keypair object to store in the keystore.
 
         :return: The newly added key object.
         """
-
+        session = session or self._session_on_init_thread
         fingerprint = fingerprint_from_key(key)
         key_data = bytes(key)
-
         new_key = Key(fingerprint, key_data, is_signing)
 
-        self.session.add(new_key)
-        self.session.commit()
+        session.add(new_key)
+        session.commit()
+
         return new_key
 
-    def get_key(self, fingerprint: bytes) -> Union[keypairs.EncryptingKeypair,
+    def get_key(self, fingerprint: bytes, session=None) -> Union[keypairs.EncryptingKeypair,
                                                    keypairs.SigningKeypair]:
         """
         Returns a key from the KeyStore.
@@ -59,7 +63,10 @@ class KeyStore(object):
 
         :return: Keypair of the returned key.
         """
-        key = self.session.query(Key).filter_by(fingerprint=fingerprint).first()
+        session = session or self._session_on_init_thread
+
+        key = session.query(Key).filter_by(fingerprint=fingerprint).first()
+
         if not key:
             raise NotFound(
                 "No key with fingerprint {} found.".format(fingerprint))
@@ -67,81 +74,110 @@ class KeyStore(object):
         pubkey = UmbralPublicKey.from_bytes(key.key_data, as_b64=False)
         return pubkey
 
-    def del_key(self, fingerprint: bytes):
+    def del_key(self, fingerprint: bytes, session=None):
         """
         Deletes a key from the KeyStore.
 
         :param fingerprint: Fingerprint of key to delete
         """
-        self.session.query(Key).filter_by(fingerprint=fingerprint).delete()
-        self.session.commit()
+        session = session or self._session_on_init_thread
+
+        session.query(Key).filter_by(fingerprint=fingerprint).delete()
+        session.commit()
 
     def add_policy_contract(self, expiration, deposit, hrac, kfrag=None,
                             alice_pubkey_sig=None, # alice_pubkey_enc,
-                            alice_signature=None) -> PolicyContract:
+                            alice_signature=None, session=None) -> PolicyContract:
         """
         Creates a PolicyContract to the Keystore.
 
         :return: The newly added PolicyContract object
         """
-        # TODO: This can be optimized to one commit/write.
-        alice_pubkey_sig = self.add_key(alice_pubkey_sig, is_signing=True)
+        session = session or self._session_on_init_thread
+
+        alice_key_instance = session.query(Key).filter_by(key_data=bytes(alice_pubkey_sig)).first()
+        if not alice_key_instance:
+            alice_key_instance = Key.from_umbral_key(alice_pubkey_sig, is_signing=True)
         # alice_pubkey_enc = self.add_key(alice_pubkey_enc)
         # bob_pubkey_sig = self.add_key(bob_pubkey_sig)
 
         new_policy_contract = PolicyContract(
-            expiration, deposit, hrac, kfrag, alice_pubkey_sig=alice_pubkey_sig,
+            expiration, deposit, hrac, kfrag, alice_pubkey_sig=alice_key_instance,
             alice_signature=None, # bob_pubkey_sig.id
         )
 
-        self.session.add(new_policy_contract)
-        self.session.commit()
+        session.add(new_policy_contract)
+        session.commit()
 
         return new_policy_contract
 
-    def get_policy_contract(self, hrac: bytes) -> PolicyContract:
+    def get_policy_contract(self, hrac: bytes, session=None) -> PolicyContract:
         """
         Returns the PolicyContract by its HRAC.
 
         :return: The PolicyContract object
         """
-        policy_contract = self.session.query(PolicyContract).filter_by(hrac=hrac).first()
+        session = session or self._session_on_init_thread
+
+        policy_contract = session.query(PolicyContract).filter_by(hrac=hrac).first()
+
         if not policy_contract:
             raise NotFound("No PolicyContract with {} HRAC found.".format(hrac))
         return policy_contract
 
-    def del_policy_contract(self, hrac: bytes):
+    def del_policy_contract(self, hrac: bytes, session=None):
         """
         Deletes a PolicyContract from the Keystore.
         """
-        self.session.query(PolicyContract).filter_by(hrac=hrac).delete()
-        self.session.commit()
+        session = session or self._session_on_init_thread
 
-    def add_workorder(self, bob_pubkey_sig, bob_signature, hrac) -> Workorder:
+        session.query(PolicyContract).filter_by(hrac=hrac).delete()
+        session.commit()
+
+    def attach_kfrag_to_saved_contract(self, alice, hrac_as_hex, kfrag, session=None):
+        session = session or self._session_on_init_thread
+
+        policy_contract = session.query(PolicyContract).filter_by(hrac=hrac_as_hex.encode()).first()
+
+        if policy_contract.alice_pubkey_sig.key_data != alice.stamp:
+            raise alice.SuspiciousActivity
+
+        policy_contract.k_frag = bytes(kfrag)
+        session.commit()
+
+    def add_workorder(self, bob_pubkey_sig, bob_signature, hrac, session=None) -> Workorder:
         """
         Adds a Workorder to the keystore.
         """
+        session = session or self._session_on_init_thread
         bob_pubkey_sig = self.add_key(bob_pubkey_sig)
         new_workorder = Workorder(bob_pubkey_sig.id, bob_signature, hrac)
 
-        self.session.add(new_workorder)
-        self.session.commit()
+        session.add(new_workorder)
+        session.commit()
+
         return new_workorder
 
-    def get_workorders(self, hrac: bytes) -> Workorder:
+    def get_workorders(self, hrac: bytes, session=None) -> Workorder:
         """
         Returns a list of Workorders by HRAC.
         """
-        workorders = self.session.query(Workorder).filter_by(hrac=hrac)
+        session = session or self._session_on_init_thread
+
+        workorders = session.query(Workorder).filter_by(hrac=hrac)
+
         if not workorders:
             raise NotFound("No Workorders with {} HRAC found.".format(hrac))
         return workorders
 
-    def del_workorders(self, hrac: bytes):
+    def del_workorders(self, hrac: bytes, session=None):
         """
         Deletes a Workorder from the Keystore.
         """
-        workorders = self.session.query(Workorder).filter_by(hrac=hrac)
+        session = session or self._session_on_init_thread
+
+        workorders = session.query(Workorder).filter_by(hrac=hrac)
         deleted = workorders.delete()
-        self.session.commit()
+        session.commit()
+
         return deleted

nkms/keystore/threading.py

@@ -0,0 +1,15 @@
+from sqlalchemy.orm import sessionmaker, scoped_session
+
+
+class ThreadedSession:
+
+    def __init__(self, sqlalchemy_engine):
+        self.engine = sqlalchemy_engine
+
+    def __enter__(self):
+        session_factory = sessionmaker(bind=self.engine)
+        self.session = scoped_session(session_factory)
+        return self.session
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.session.remove()

nkms/network/node.py

@@ -1,8 +1,11 @@
+import requests
 from kademlia.node import Node
 
 from nkms.crypto.constants import CFRAG_LENGTH
+from nkms.crypto.kits import MessageKit
 from nkms.crypto.utils import RepeatingBytestringSplitter
 from nkms.network.capabilities import ServerCapability
+
 from umbral.fragments import CapsuleFrag
 
 
@@ -35,11 +38,24 @@ class NetworkyStuff(object):
         pass
 
     def reencrypt(self, work_order):
-        ursula = self.get_ursula_by_id(work_order.ursula_id)
-        ursula_rest_response = self.send_work_order_payload_to_ursula(work_order, ursula)
+        ursula_rest_response = self.send_work_order_payload_to_ursula(work_order)
         cfrags = RepeatingBytestringSplitter((CapsuleFrag, CFRAG_LENGTH))(ursula_rest_response.content)
         work_order.complete(cfrags)  # TODO: We'll do verification of Ursula's signature here.  #141
         return cfrags
 
     def get_competitive_rate(self):
         return NotImplemented
+
+    def get_treasure_map_from_node(self, node, map_id):
+        response = requests.get("{}/treasure_map/{}".format(node.rest_url(), map_id.hex()))
+        return response
+
+    def push_treasure_map_to_node(self, node, map_id, map_payload):
+        response = requests.post("{}/treasure_map/{}".format(node.rest_url(), map_id.hex()),
+                      data=map_payload)
+        return response
+
+    def send_work_order_payload_to_ursula(self, work_order):
+        payload = work_order.payload()
+        hrac_as_hex = work_order.kfrag_hrac.hex()
+        return requests.post('{}/kFrag/{}/reencrypt'.format(work_order.ursula.rest_url(), hrac_as_hex), payload)

nkms/network/protocols.py

@@ -49,8 +49,14 @@ class NuCypherHashProtocol(KademliaProtocol):
 
     def determine_legality_of_dht_key(self, signature, sender_pubkey_sig,
                                       message, hrac, dht_key, dht_value):
-        proper_key = digest(
-            keccak_digest(bytes(sender_pubkey_sig) + bytes(hrac)))
+
+        # TODO: This function can use a once-over.
+        # TODO: Push the logic of this if branch down.
+        if dht_value[:2] == BYTESTRING_IS_URSULA_IFACE_INFO:
+            proper_key = digest(bytes(sender_pubkey_sig))
+        else:
+            proper_key = digest(
+                keccak_digest(bytes(sender_pubkey_sig) + bytes(hrac)))
 
         verified = signature.verify(hrac, sender_pubkey_sig)
 
@@ -69,6 +75,7 @@ class NuCypherHashProtocol(KademliaProtocol):
         self.welcomeIfNewNode(source)
         self.log.debug("got a store request from %s" % str(sender))
 
+        # TODO: Why is this logic here?  This is madness.  See #172.
         if value.startswith(BYTESTRING_IS_URSULA_IFACE_INFO) or value.startswith(
                 BYTESTRING_IS_TREASURE_MAP):
             header, signature, sender_pubkey_sig, hrac, message = dht_value_splitter(

nkms/network/server.py

@@ -7,18 +7,23 @@ from apistar.http import Response
 from kademlia.crawling import NodeSpiderCrawl
 from kademlia.network import Server
 from kademlia.utils import digest
-from sqlalchemy.exc import IntegrityError
 
 from nkms.crypto.kits import MessageKit
 from nkms.crypto.powers import EncryptingPower, SigningPower
 from nkms.crypto.utils import BytestringSplitter
+from nkms.keystore.threading import ThreadedSession
 from nkms.network.capabilities import SeedOnly, ServerCapability
 from nkms.network.node import NuCypherNode
-from nkms.network.protocols import NuCypherSeedOnlyProtocol, NuCypherHashProtocol
+from nkms.network.protocols import NuCypherSeedOnlyProtocol, NuCypherHashProtocol, \
+    dht_value_splitter
 from nkms.network.storage import SeedOnlyStorage
 from umbral import pre
 from umbral.fragments import KFrag
 
+from apistar.core import Route
+from apistar.frameworks.wsgi import WSGIApp as App
+
+
 
 class NuCypherDHTServer(Server):
     protocol_class = NuCypherHashProtocol
@@ -94,34 +99,82 @@ class NuCypherSeedOnlyDHTServer(NuCypherDHTServer):
 
 class ProxyRESTServer(object):
 
+    def __init__(self, rest_address, rest_port, db_name):
+        self.rest_address = rest_address
+        self.rest_port = rest_port
+        self.db_name = db_name
+        self._rest_app = None
+
+    def attach_rest_server(self, db_name):
+
+        routes = [
+            Route('/kFrag/{hrac_as_hex}',
+                  'POST',
+                  self.set_policy),
+            Route('/kFrag/{hrac_as_hex}/reencrypt',
+                  'POST',
+                  self.reencrypt_via_rest),
+            Route('/public_keys', 'GET',
+                  self.get_signing_and_encrypting_public_keys),
+            Route('/consider_contract',
+                  'POST',
+                  self.consider_contract),
+            Route('/treasure_map/{treasure_map_id_as_hex}',
+                  'GET',
+                  self.provide_treasure_map),
+            Route('/treasure_map/{treasure_map_id_as_hex}',
+                  'POST',
+                  self.receive_treasure_map),
+        ]
+
+        self._rest_app = App(routes=routes)
+        self.start_datastore(db_name)
+
+    def start_datastore(self, db_name):
+        if not db_name:
+            raise TypeError("In order to start a datastore, you need to supply a db_name.")
+
+        from nkms.keystore import keystore
+        from nkms.keystore.db import Base
+        from sqlalchemy.engine import create_engine
+
+        engine = create_engine('sqlite:///{}'.format(db_name))
+        Base.metadata.create_all(engine)
+        self.datastore = keystore.KeyStore(engine)
+        self.db_engine = engine
+
+    def rest_url(self):
+        return "{}:{}".format(self.rest_address, self.rest_port)
+
+    # """
+    # Actual REST Endpoints and utilities
+    # """
+    # def find_ursulas_by_ids(self, request: http.Request):
+    #
+    #
+
     def get_signing_and_encrypting_public_keys(self):
         """
         REST endpoint for getting both signing and encrypting public keys.
         """
         return Response(
-            content=bytes(self.stamp) + bytes(self.public_key(EncryptingPower)),
+            content=bytes(self.public_key(SigningPower)) + bytes(self.public_key(EncryptingPower)),
             content_type="application/octet-stream")
 
     def consider_contract(self, hrac_as_hex, request: http.Request):
-        # TODO: This actually needs to be a REST endpoint, with the payload
-        # carrying the kfrag hash separately.
         from nkms.policy.models import Contract
         contract, deposit_as_bytes = \
             BytestringSplitter(Contract)(request.body, return_remainder=True)
         contract.deposit = deposit_as_bytes
 
-        # contract_to_store = {  # TODO: This needs to be a datastore - see #127.
-        #     "alice_pubkey_sig":
-        #     "deposit": contract.deposit,
-        #     # TODO: Whatever type "deposit" ends up being, we'll need to
-        #     # serialize it here.  See #148.
-        #     "expiration": contract.expiration,
-        # }
-        self.keystore.add_policy_contract(contract.expiration.datetime(),
-                                          contract.deposit,
-                                          hrac=contract.hrac.hex().encode(),
-                                          alice_pubkey_sig=contract.alice.stamp
-                                          )
+        with ThreadedSession(self.db_engine) as session:
+            self.datastore.add_policy_contract(
+                contract.expiration.datetime(),
+                contract.deposit,
+                hrac=contract.hrac.hex().encode(),
+                alice_pubkey_sig=contract.alice.stamp,
+                session=session,
+                )
         # TODO: Make the rest of this logic actually work - do something here
         # to decide if this Contract is worth accepting.
         return Response(
@@ -141,7 +194,7 @@ class ProxyRESTServer(object):
         # group_payload_splitter = BytestringSplitter(PublicKey)
         # policy_payload_splitter = BytestringSplitter((KFrag, KFRAG_LENGTH))
 
-        alice = self._alice_class.from_public_keys((SigningPower, policy_message_kit.alice_pubkey))
+        alice = self._alice_class.from_public_keys({SigningPower: policy_message_kit.alice_pubkey})
 
         verified, cleartext = self.verify_from(
             alice, policy_message_kit,
@@ -159,24 +212,12 @@ class ProxyRESTServer(object):
         # kfrag = policy_payload_splitter(policy_payload)[0]
         kfrag = KFrag.from_bytes(cleartext)
 
-        # TODO: Query stored Contract and reconstitute
-        policy_contract = self.keystore.get_policy_contract(hrac_as_hex.encode())
-        # contract_details = self._contracts[hrac.hex()]
-
-        if policy_contract.alice_pubkey_sig.key_data != alice.stamp:
-            raise self._alice_class.SuspiciousActivity
-
-        # contract = Contract(alice=alice, hrac=hrac,
-        #                     kfrag=kfrag, expiration=policy_contract.expiration)
-
-        try:
-            # TODO: Obviously we do this lower-level.
-            policy_contract.k_frag = bytes(kfrag)
-            self.keystore.session.commit()
-
-        except IntegrityError:
-            raise
-            # Do something appropriately RESTful (ie, 4xx).
+        with ThreadedSession(self.db_engine) as session:
+            self.datastore.attach_kfrag_to_saved_contract(
+                                               alice,
+                                               hrac_as_hex,
+                                               kfrag,
+                                               session=session)
 
         return  # TODO: Return A 200, with whatever policy metadata.
 
@@ -184,7 +225,9 @@ class ProxyRESTServer(object):
         from nkms.policy.models import WorkOrder  # Avoid circular import
         hrac = binascii.unhexlify(hrac_as_hex)
         work_order = WorkOrder.from_rest_payload(hrac, request.body)
-        kfrag_bytes = self.keystore.get_policy_contract(hrac.hex().encode()).k_frag  # Careful!  :-)
+        with ThreadedSession(self.db_engine) as session:
+            kfrag_bytes = self.datastore.get_policy_contract(hrac.hex().encode(),
+                                                             session=session).k_frag  # Careful!  :-)
         # TODO: Push this to a lower level.
         kfrag = KFrag.from_bytes(kfrag_bytes)
         cfrag_byte_stream = b""
@@ -198,3 +241,30 @@ class ProxyRESTServer(object):
 
         return Response(content=cfrag_byte_stream,
                         content_type="application/octet-stream")
+
+    def provide_treasure_map(self, treasure_map_id_as_hex):
+        # For now, grab the TreasureMap for the DHT storage.  Soon, no do that.  #TODO!
+        treasure_map_id = binascii.unhexlify(treasure_map_id_as_hex)
+        treasure_map_bytes = self.server.storage.get(digest(treasure_map_id))
+        return Response(content=treasure_map_bytes,
+                        content_type="application/octet-stream")
+
+    def receive_treasure_map(self, treasure_map_id_as_hex, request: http.Request):
+        # TODO: This function is the epitome of #172.
+        treasure_map_id = binascii.unhexlify(treasure_map_id_as_hex)
+
+        header, signature_for_ursula, pubkey_sig_alice, hrac, tmap_message_kit = \
+            dht_value_splitter(request.body, return_remainder=True)
+        # TODO: This next line is possibly the worst in the entire codebase at the moment.  #172.
+        # Also TODO: TTL?
+        do_store = self.server.protocol.determine_legality_of_dht_key(signature_for_ursula, pubkey_sig_alice, tmap_message_kit,
+                                                      hrac, digest(treasure_map_id), request.body)
+        if do_store:
+            # TODO: Stop storing things in the protocol storage.  Do this better.
+            # TODO: Propagate to other nodes.
+            self.server.protocol.storage[digest(treasure_map_id)] = request.body
+            return # TODO: Proper response here.
+        else:
+            # TODO: Make this a proper 500 or whatever.
+            assert False
+

nkms/policy/models.py

@@ -1,5 +1,6 @@
 import asyncio
 import binascii
+from collections import OrderedDict
 
 import maya
 import msgpack
@@ -57,7 +58,7 @@ class Contract(object):
         alice_pubkey_sig, hrac, expiration_bytes, deposit_bytes = contract_splitter(
             contract_as_bytes, return_remainder=True)
         expiration = maya.parse(expiration_bytes.decode())
-        alice = Alice.from_public_keys((SigningPower, alice_pubkey_sig))
+        alice = Alice.from_public_keys({SigningPower: alice_pubkey_sig})
         return cls(alice=alice, hrac=hrac, expiration=expiration)
 
     def activate(self, kfrag, ursula, negotiation_result):
@@ -108,7 +109,7 @@ class Policy(object):
         self.uri = uri
         self.m = m
         self.treasure_map = TreasureMap()
-        self._accepted_contracts = {}
+        self._accepted_contracts = OrderedDict()
 
         self.alices_signature = alices_signature
 
@@ -165,11 +166,7 @@ class Policy(object):
         Alice and Bob have all the information they need to construct this.
         Ursula does not, so we share it with her.
         """
-        return Policy.hash(bytes(alice.stamp) + bytes(bob.stamp) + uri)
-
-    @staticmethod
-    def hash(message):
-        return keccak_digest(message)
+        return keccak_digest(bytes(alice.stamp) + bytes(bob.stamp) + uri)
 
     def treasure_map_dht_key(self):
         """
@@ -179,9 +176,11 @@ class Policy(object):
 
         Our public key (which everybody knows) and the hrac above.
         """
-        return self.hash(bytes(self.alice.stamp) + self.hrac())
+        return keccak_digest(bytes(self.alice.stamp) + self.hrac())
 
-    def publish_treasure_map(self):
+    def publish_treasure_map(self, networky_stuff=None, use_dht=True):
+        if networky_stuff is None and use_dht is False:
+            raise ValueError("Can't engage the REST swarm without networky stuff.")
         tmap_message_kit, signature_for_bob = self.alice.encrypt_for(
             self.bob,
             self.treasure_map.packed_payload())
@@ -189,13 +188,21 @@ class Policy(object):
 
         # In order to know this is safe to propagate, Ursula needs to see a signature, our public key,
         # and, reasons explained in treasure_map_dht_key above, the uri_hash.
-        dht_value = signature_for_ursula + self.alice.stamp + self.hrac() + tmap_message_kit.to_bytes()
-        dht_key = self.treasure_map_dht_key()
+        # TODO: Clean this up.  See #172.
+        map_payload = signature_for_ursula + self.alice.stamp + self.hrac() + tmap_message_kit.to_bytes()
+        map_id = self.treasure_map_dht_key()
 
-        setter = self.alice.server.set(dht_key, BYTESTRING_IS_TREASURE_MAP + dht_value)
-        event_loop = asyncio.get_event_loop()
-        event_loop.run_until_complete(setter)
-        return tmap_message_kit, dht_value, signature_for_bob, signature_for_ursula
+        if use_dht:
+            setter = self.alice.server.set(map_id, BYTESTRING_IS_TREASURE_MAP + map_payload)
+            event_loop = asyncio.get_event_loop()
+            event_loop.run_until_complete(setter)
+        else:
+            for node in self.alice.known_nodes.values():
+                response = networky_stuff.push_treasure_map_to_node(node, map_id, BYTESTRING_IS_TREASURE_MAP + map_payload)
+                # TODO: Do something here based on success or failure
+                if response.status_code == 204:
+                    pass
+        return tmap_message_kit, map_payload, signature_for_bob, signature_for_ursula
 
     def enact(self, networky_stuff):
         for contract in self._accepted_contracts.values():
@@ -271,19 +278,19 @@ class TreasureMap(object):
 
 class WorkOrder(object):
     def __init__(self, bob, kfrag_hrac, capsules, receipt_bytes,
-                 receipt_signature, ursula_id=None):
+                 receipt_signature, ursula=None):
         self.bob = bob
         self.kfrag_hrac = kfrag_hrac
         self.capsules = capsules
         self.receipt_bytes = receipt_bytes
         self.receipt_signature = receipt_signature
-        self.ursula_id = ursula_id  # TODO: We may still need a more elegant system for ID'ing Ursula.  See #136.
+        self.ursula = ursula  # TODO: We may still need a more elegant system for ID'ing Ursula.  See #136.
 
     def __repr__(self):
         return "WorkOrder for hrac {hrac}: (capsules: {capsule_bytes}) for {ursulas}".format(
             hrac=self.kfrag_hrac.hex()[:6],
             capsule_bytes=[binascii.hexlify(bytes(cap))[:6] for cap in self.capsules],
-            ursulas=binascii.hexlify(bytes(self.ursula_id))[:6])
+            ursulas=binascii.hexlify(bytes(self.ursula.stamp))[:6])
 
     def __eq__(self, other):
         return (self.receipt_bytes, self.receipt_signature) == (
@@ -293,11 +300,11 @@ class WorkOrder(object):
         return len(self.capsules)
 
     @classmethod
-    def construct_by_bob(cls, kfrag_hrac, capsules, ursula_dht_key, bob):
-        receipt_bytes = b"wo:" + ursula_dht_key  # TODO: represent the capsules as bytes and hash them as part of the receipt, ie  + keccak_digest(b"".join(capsules))  - See #137
+    def construct_by_bob(cls, kfrag_hrac, capsules, ursula, bob):
+        receipt_bytes = b"wo:" + ursula.interface_dht_key()  # TODO: represent the capsules as bytes and hash them as part of the receipt, ie  + keccak_digest(b"".join(capsules))  - See #137
         receipt_signature = bob.stamp(receipt_bytes)
         return cls(bob, kfrag_hrac, capsules, receipt_bytes, receipt_signature,
-                   ursula_dht_key)
+                   ursula)
 
     @classmethod
     def from_rest_payload(cls, kfrag_hrac, rest_payload):
@@ -308,7 +315,7 @@ class WorkOrder(object):
         verified = signature.verify(receipt_bytes, bob_pubkey_sig)
         if not verified:
             raise ValueError("This doesn't appear to be from Bob.")
-        bob = Bob.from_public_keys((SigningPower, bob_pubkey_sig))
+        bob = Bob.from_public_keys({SigningPower: bob_pubkey_sig})
         return cls(bob, kfrag_hrac, capsules, receipt_bytes, signature)
 
     def payload(self):
@@ -332,10 +339,10 @@ class WorkOrderHistory:
         assert False
 
     def __getitem__(self, item):
-        if isinstance(item, Ursula.InterfaceDHTKey):
+        if isinstance(item, bytes):
             return self.by_ursula.setdefault(item, {})
         else:
-            raise TypeError("If you want to lookup a WorkOrder by Ursula, you need to pass an Ursula.InterfaceDHTKey.")
+            raise TypeError("If you want to lookup a WorkOrder by Ursula, you need to pass bytes of her signing public key.")
 
     def __setitem__(self, key, value):
         assert False

tests/characters/test_alice_can_grant_and_revoke.py

@@ -28,7 +28,7 @@ def test_grant(alice, bob, ursulas):
 
     # Get the Policy from Ursula's datastore, looking up by hrac.
     proper_hrac = keccak_digest(bytes(alice.stamp) + bytes(bob.stamp) + uri)
-    retrieved_policy = ursula.keystore.get_policy_contract(proper_hrac.hex().encode())
+    retrieved_policy = ursula.datastore.get_policy_contract(proper_hrac.hex().encode())
 
     # TODO: Make this a legit KFrag, not bytes.
     retrieved_k_frag = KFrag.from_bytes(retrieved_policy.k_frag)
@@ -49,8 +49,7 @@ def test_alice_can_get_ursulas_keys_via_rest(alice, ursulas):
         (UmbralPublicKey, PUBLIC_KEY_LENGTH, {"as_b64": False})
     )
     signing_key, encrypting_key = splitter(response.content)
-    stranger_ursula_from_public_keys = Ursula.from_public_keys((SigningPower,
-                                                                signing_key),
-                                                               (EncryptingPower,
-                                                                encrypting_key))
+    stranger_ursula_from_public_keys = Ursula.from_public_keys({SigningPower: signing_key,
+                                                               EncryptingPower: encrypting_key}
+                                                            )
     assert stranger_ursula_from_public_keys == ursulas[0]

tests/characters/test_bob_handles_frags.py

@@ -16,13 +16,13 @@ def test_bob_can_follow_treasure_map(enacted_policy, ursulas, alice, bob):
     bob.treasure_maps[hrac] = treasure_map
 
     # Bob knows of no Ursulas.
-    assert len(bob._ursulas) == 0
+    assert len(bob.known_nodes) == 0
 
     # ...until he follows the TreasureMap.
     bob.follow_treasure_map(hrac)
 
     # Now he knows of all the Ursulas.
-    assert len(bob._ursulas) == len(treasure_map)
+    assert len(bob.known_nodes) == len(treasure_map)
 
 
 def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice, bob, ursulas,
@@ -39,7 +39,7 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice,
     hrac, treasure_map = enacted_policy.hrac(), enacted_policy.treasure_map
     bob.treasure_maps[hrac] = treasure_map
     bob.follow_treasure_map(hrac)
-    assert len(bob._ursulas) == len(ursulas)
+    assert len(bob.known_nodes) == len(ursulas)
 
     the_hrac = enacted_policy.hrac()
 
@@ -61,6 +61,9 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice,
     networky_stuff = MockNetworkyStuff(ursulas)
     ursula_dht_key, work_order = list(work_orders.items())[0]
 
+    # In the real world, we'll have a full Ursula node here.  But in this case, we need to fake it.
+    work_order.ursula = ursulas[0]
+
     # **** RE-ENCRYPTION HAPPENS HERE! ****
     cfrags = bob.get_reencrypted_c_frags(networky_stuff, work_order)
 
@@ -76,8 +79,8 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice,
 
     # OK, so cool - Bob has his cFrag!  Let's make sure everything went properly.  First, we'll show that it is in fact
     # the correct cFrag (ie, that Ursula performed reencryption properly).
-    ursula = networky_stuff.get_ursula_by_id(work_order.ursula_id)
-    kfrag_bytes = ursula.keystore.get_policy_contract(
+    ursula = work_order.ursula
+    kfrag_bytes = ursula.datastore.get_policy_contract(
         work_order.kfrag_hrac.hex().encode()).k_frag
     the_kfrag = KFrag.from_bytes(kfrag_bytes)
     the_correct_cfrag = pre.reencrypt(the_kfrag, alicebob_side_channel.capsule)
@@ -121,6 +124,9 @@ def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_polic
 
     # We can get a new CFrag, just like last time.
     networky_stuff = MockNetworkyStuff(ursulas)
+    # In the real world, we'll have a full Ursula node here.  But in this case, we need to fake it.
+    new_work_order.ursula = ursulas[1]
+
     cfrags = bob.get_reencrypted_c_frags(networky_stuff, new_work_order)
 
     # Again: one Capsule, one cFrag.
@@ -150,6 +156,8 @@ def test_bob_gathers_and_combines(enacted_policy, alice, bob, ursulas, alicebob_
     _id_of_yet_another_ursula, new_work_order = list(new_work_orders.items())[0]
 
     networky_stuff = MockNetworkyStuff(ursulas)
+    # In the real world, we'll have a full Ursula node here.  But in this case, we need to fake it.
+    new_work_order.ursula = ursulas[2]
     cfrags = bob.get_reencrypted_c_frags(networky_stuff, new_work_order)
     alicebob_side_channel.capsule.attach_cfrag(cfrags[0])
 

tests/characters/test_crypto_characters_and_their_powers.py

@@ -84,14 +84,13 @@ def test_anybody_can_encrypt():
     """
     Similar to anybody_can_verify() above; we show that anybody can encrypt.
     """
-    can_sign_and_encrypt = Character(crypto_power_ups=[SigningPower, EncryptingPower])
-    ursula = Ursula()
+    everyman = Character()
+    ursula = Ursula(is_me=False)
 
     cleartext = b"This is Officer Rod Farva. Come in, Ursula!  Come in Ursula!"
 
-    ciphertext, signature = can_sign_and_encrypt.encrypt_for(ursula, cleartext, sign=False)
+    ciphertext, signature = everyman.encrypt_for(ursula, cleartext, sign=False)
     assert signature == NOT_SIGNED
-
     assert ciphertext is not None
 
 """
@@ -132,6 +131,10 @@ def test_encrypt_and_sign_the_ciphertext(alice, bob):
 
 
 def test_encrypt_but_do_not_sign(alice, bob):
+    """
+    Finally, Alice encrypts but declines to sign.
+    This is useful in a scenario in which Alice wishes to plausibly disavow having created this content.
+    """
     message = b"If Bonnie comes home and finds an unencrypted private key in her keystore, I'm gonna get divorced."
 
     # Alice might also want to encrypt a message but *not* sign it, in order to refrain
@@ -148,5 +151,3 @@ def test_encrypt_but_do_not_sign(alice, bob):
 
     # However, the message was properly decrypted.
     assert message == cleartext
-
-

tests/fixtures.py

@@ -1,5 +1,5 @@
 import datetime
-
+import os
 import pytest
 
 from nkms.characters import Alice, Bob
@@ -50,7 +50,6 @@ def enacted_policy(idle_policy, ursulas):
 @pytest.fixture(scope="module")
 def alice(ursulas):
     ALICE = Alice()
-    ALICE.attach_server()
     ALICE.server.listen(8471)
     ALICE.__resource_id = b"some_resource_id"
     EVENT_LOOP.run_until_complete(ALICE.server.bootstrap([("127.0.0.1", u.dht_port) for u in ursulas]))
@@ -60,7 +59,6 @@ def alice(ursulas):
 @pytest.fixture(scope="module")
 def bob(alice, ursulas):
     BOB = Bob(alice=alice)
-    BOB.attach_server()
     BOB.server.listen(8475)
     EVENT_LOOP.run_until_complete(BOB.server.bootstrap([("127.0.0.1", URSULA_PORT)]))
     return BOB
@@ -70,6 +68,10 @@ def bob(alice, ursulas):
 def ursulas():
     URSULAS = make_ursulas(NUMBER_OF_URSULAS_IN_NETWORK, URSULA_PORT)
     yield URSULAS
+    # Remove the DBs that have been sprayed hither and yon.
+    for _u in range(NUMBER_OF_URSULAS_IN_NETWORK):
+        port = URSULA_PORT + _u
+        os.remove("test-{}".format(port))
     blockchain_client._ursulas_on_blockchain.clear()
 
 

tests/network/test_network_actors.py

@@ -5,9 +5,11 @@ import pytest
 
 from kademlia.utils import digest
 from nkms.characters import Ursula, Character
+from nkms.crypto.api import keccak_digest
 from nkms.crypto.kits import MessageKit
 from nkms.network import blockchain_client
 from nkms.network.constants import BYTESTRING_IS_TREASURE_MAP, BYTESTRING_IS_URSULA_IFACE_INFO
+
 from nkms.network.protocols import dht_value_splitter
 
 from tests.utilities import MockNetworkyStuff, EVENT_LOOP, URSULA_PORT, NUMBER_OF_URSULAS_IN_NETWORK
@@ -72,7 +74,7 @@ def test_alice_creates_policy_group_with_correct_hrac(idle_policy):
     alice = idle_policy.alice
     bob = idle_policy.bob
 
-    assert idle_policy.hrac() == idle_policy.hash(
+    assert idle_policy.hrac() == keccak_digest(
         bytes(alice.stamp) + bytes(bob.stamp) + alice.__resource_id)
 
 
@@ -140,7 +142,16 @@ def test_bob_can_retreive_the_treasure_map_and_decrypt_it(enacted_policy, ursula
 
     # Of course, in the real world, Bob has sufficient information to reconstitute a PolicyGroup, gleaned, we presume,
     # through a side-channel with Alice.
-    treasure_map_from_wire = bob.get_treasure_map(enacted_policy)
+
+    # If Bob doesn't know about any Ursulas, he can't find the TreasureMap via the REST swarm:
+    with pytest.raises(bob.NotEnoughUrsulas):
+        treasure_map_from_wire = bob.get_treasure_map(enacted_policy, networky_stuff)
+
+    # Let's imagine he has learned about some - say, from the blockchain.
+    bob.known_nodes = {u.interface_dht_key(): u for u in ursulas}
+
+    # Now try.
+    treasure_map_from_wire = bob.get_treasure_map(enacted_policy, networky_stuff)
 
     assert enacted_policy.treasure_map == treasure_map_from_wire
 

tests/network/test_network_upgrade.py

@@ -4,5 +4,5 @@ def test_alice_enacts_policies_in_policy_group_via_rest(enacted_policy):
     and transmitting them via REST.
     """
     ursula = list(enacted_policy._accepted_contracts.values())[0].ursula
-    policy_contract = ursula.keystore.get_policy_contract(enacted_policy.hrac().hex().encode())
+    policy_contract = ursula.datastore.get_policy_contract(enacted_policy.hrac().hex().encode())
     assert bool(policy_contract)  # TODO: This can be a more poignant assertion.

tests/utilities.py

@@ -30,12 +30,15 @@ def make_ursulas(how_many_ursulas: int, ursula_starting_port: int) -> list:
 
     URSULAS = []
     for _u in range(how_many_ursulas):
-        engine = create_engine('sqlite:///:memory:')
-        Base.metadata.create_all(engine)
-        ursulas_keystore = keystore.KeyStore(engine)
-        _URSULA = Ursula(urulsas_keystore=ursulas_keystore)
-        _URSULA.attach_server()
-        _URSULA.listen(ursula_starting_port + _u, "127.0.0.1")
+        port = ursula_starting_port + _u
+        _URSULA = Ursula(dht_port=port, dht_interface="127.0.0.1", db_name="test-{}".format(port))
+
+        class MockDatastoreThreadPool(object):
+            def callInThread(self, f, *args, **kwargs):
+                return f(*args, **kwargs)
+
+        _URSULA.datastore_threadpool = MockDatastoreThreadPool()
+        _URSULA.listen()
 
         URSULAS.append(_URSULA)
 
@@ -76,15 +79,13 @@ class MockNetworkyStuff(NetworkyStuff):
         response = mock_client.post('http://localhost/kFrag/{}'.format(hrac.hex()), payload)
         return True, ursula.interface_dht_key()
 
-    def get_ursula_by_id(self, ursula_id):
-        try:
-            ursula = self._ursulas[ursula_id]
-        except KeyError:
-            pytest.fail("No Ursula with ID {}".format(ursula_id))
-        return ursula
-
-    def send_work_order_payload_to_ursula(self, work_order, ursula):
-        mock_client = TestClient(ursula.rest_app)
+    def send_work_order_payload_to_ursula(self, work_order):
+        mock_client = TestClient(work_order.ursula.rest_app)
         payload = work_order.payload()
         hrac_as_hex = work_order.kfrag_hrac.hex()
         return mock_client.post('http://localhost/kFrag/{}/reencrypt'.format(hrac_as_hex), payload)
+
+    def get_treasure_map_from_node(self, node, map_id):
+        mock_client = TestClient(node.rest_app)
+        return mock_client.get("http://localhost/treasure_map/{}".format(map_id.hex()))
+