From 13ff2e642c9b9b2b3cce26e2bdc24e27ae2bd432 Mon Sep 17 00:00:00 2001 From: jMyles Date: Sat, 14 Apr 2018 09:05:11 -0700 Subject: [PATCH 01/15] Actors can now discover all known nodes from a given node. Fixes #175. --- .../alicebob-grant-and-line-by-line-PRE.py | 25 +---------------- ..._ursula_with_rest_and_dht_but_no_mining.py | 0 examples/sandbox_resources.py | 0 nkms/characters.py | 27 ++++++++++++++----- nkms/crypto/kits.py | 7 ++++- nkms/network/node.py | 8 +++--- nkms/network/protocols.py | 7 +++++ nkms/network/server.py | 16 +++++++---- nkms/policy/models.py | 10 +++---- tests/network/test_network_actors.py | 11 +++++++- tests/utilities.py | 24 ++++++++++++++++- 11 files changed, 88 insertions(+), 47 deletions(-) rename {entry_points => examples}/run_ursula_with_rest_and_dht_but_no_mining.py (100%) create mode 100644 examples/sandbox_resources.py diff --git a/examples/alicebob-grant-and-line-by-line-PRE.py b/examples/alicebob-grant-and-line-by-line-PRE.py index f0fda4311..b9c906919 100644 --- a/examples/alicebob-grant-and-line-by-line-PRE.py +++ b/examples/alicebob-grant-and-line-by-line-PRE.py @@ -6,8 +6,7 @@ import datetime import sys -import requests - +from examples.sandbox_resources import SandboxNetworkyStuff from nkms.characters import Alice, Bob, Ursula from nkms.crypto.kits import MessageKit from nkms.crypto.powers import SigningPower, EncryptingPower @@ -18,27 +17,6 @@ ALICE = Alice() BOB = Bob() URSULA = Ursula.from_rest_url(address="https://localhost", port="3550") - -class SandboxNetworkyStuff(NetworkyStuff): - def find_ursula(self, contract=None): - ursula = Ursula.as_discovered_on_network(dht_port=None, dht_interface=None, - rest_address="https://localhost", rest_port=3550, - powers_and_keys={ - SigningPower: URSULA.stamp.as_umbral_pubkey(), - EncryptingPower: URSULA.public_key(EncryptingPower) - } - ) - response = requests.post("https://localhost:3550/consider_contract", bytes(contract), verify=False) - response.was_accepted = True - return ursula, response - - def enact_policy(self, ursula, hrac, payload): - response = requests.post('{}:{}/kFrag/{}'.format(ursula.rest_address, ursula.rest_port, hrac.hex()), - payload, verify=False) - # TODO: Something useful here and it's probably ready to go down into NetworkyStuff. - return response.status_code == 200 - - networky_stuff = SandboxNetworkyStuff() policy_end_datetime = datetime.datetime.now() + datetime.timedelta(days=5) @@ -49,7 +27,6 @@ uri = b"secret/files/and/stuff" ALICE.learn_about_nodes(address="https://localhost", port="3550") # Alice grants to Bob. - policy = ALICE.grant(BOB, uri, networky_stuff, m=1, n=n, expiration=policy_end_datetime) policy.publish_treasure_map(networky_stuff, use_dht=False) diff --git a/entry_points/run_ursula_with_rest_and_dht_but_no_mining.py b/examples/run_ursula_with_rest_and_dht_but_no_mining.py similarity index 100% rename from entry_points/run_ursula_with_rest_and_dht_but_no_mining.py rename to examples/run_ursula_with_rest_and_dht_but_no_mining.py diff --git a/examples/sandbox_resources.py b/examples/sandbox_resources.py new file mode 100644 index 000000000..e69de29bb diff --git a/nkms/characters.py b/nkms/characters.py index 63f5f41ba..41bc17636 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -9,9 +9,10 @@ from kademlia.network import Server from kademlia.utils import digest from typing import Dict, ClassVar from typing import Union, List + +from bytestring_splitter import BytestringSplitter from umbral.keys import UmbralPublicKey from constant_sorrow import constants, default_constant_splitter -from bytestring_splitter import RepeatingBytestringSplitter from nkms.blockchain.eth.actors import PolicyAuthor from nkms.config.configs import KMSConfig @@ -277,13 +278,25 @@ class Character(object): power_up = self._crypto_power.power_ups(power_up_class) return power_up.public_key() - def learn_about_nodes(self, address, port): + def learn_about_nodes(self, networky_stuff, address, port): """ Sends a request to node_url to find out about known nodes. """ - # TODO: Find out about other known nodes, not just this one. #175 - node = Ursula.from_rest_url(address, port) - self.known_nodes[node.interface_dht_key()] = node + response = networky_stuff.get_nodes_via_rest(address, port) + signature, nodes = signature_splitter(response.content, return_remainder=True) + # TODO: Although not treasure map-related, this has a whiff of #172. + ursula_interface_splitter = dht_value_splitter + BytestringSplitter((bytes, 15)) + split_nodes = ursula_interface_splitter.repeat(nodes) + new_nodes = {} + for node in split_nodes: + # Notice that we don't use "interface_hrac" - see #228. + header, sig, pubkey, interface_hrac, interface = node + if sig.verify(keccak_digest(interface), pubkey): + self.known_nodes[pubkey] = msgpack.loads(interface) + new_nodes[pubkey] = msgpack.loads(interface) + else: + self.log.warn("Discovered node with bad signature: {}".format(node)) + return new_nodes class FakePolicyAgent: # TODO: #192 @@ -529,8 +542,8 @@ class Ursula(Character, ProxyRESTServer): return ursula @classmethod - def from_rest_url(cls, address, port): - response = requests.get("{}:{}/public_keys".format(address, port), verify=False) # TODO: TLS-only. + def from_rest_url(cls, networky_stuff, address, port): + response = networky_stuff.ursula_from_rest_interface(address, port) if not response.status_code == 200: raise RuntimeError("Got a bad response: {}".format(response)) diff --git a/nkms/crypto/kits.py b/nkms/crypto/kits.py index 1baf26e5b..f9bb86bc4 100644 --- a/nkms/crypto/kits.py +++ b/nkms/crypto/kits.py @@ -3,7 +3,6 @@ from constant_sorrow import constants class CryptoKit: - return_remainder_when_splitting = True splitter = None @classmethod @@ -49,8 +48,14 @@ class MessageKit(CryptoKit): class UmbralMessageKit(MessageKit): + return_remainder_when_splitting = True splitter = capsule_splitter + key_splitter def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.policy_pubkey = None + + @classmethod + def from_bytes(cls, some_bytes): + capsule, sender_pubkey_sig, ciphertext = cls.split_bytes(some_bytes) + return cls(capsule=capsule, sender_pubkey_sig=sender_pubkey_sig, ciphertext=ciphertext) diff --git a/nkms/network/node.py b/nkms/network/node.py index dbaa7ce0f..aa0faea85 100644 --- a/nkms/network/node.py +++ b/nkms/network/node.py @@ -1,8 +1,8 @@ import requests from kademlia.node import Node +from bytestring_splitter import BytestringSplitter from nkms.crypto.constants import CFRAG_LENGTH -from bytestring_splitter import RepeatingBytestringSplitter from nkms.network.capabilities import ServerCapability from umbral.fragments import CapsuleFrag @@ -38,7 +38,7 @@ class NetworkyStuff(object): def reencrypt(self, work_order): ursula_rest_response = self.send_work_order_payload_to_ursula(work_order) - cfrags = RepeatingBytestringSplitter((CapsuleFrag, CFRAG_LENGTH))(ursula_rest_response.content) + cfrags = BytestringSplitter((CapsuleFrag, CFRAG_LENGTH)).repeat(ursula_rest_response.content) work_order.complete(cfrags) # TODO: We'll do verification of Ursula's signature here. #141 return cfrags @@ -56,7 +56,9 @@ class NetworkyStuff(object): def send_work_order_payload_to_ursula(self, work_order): payload = work_order.payload() - hrac_as_hex = work_order.kfrag_hrac.hex() return requests.post('{}/kFrag/{}/reencrypt'.format(work_order.ursula.rest_url(), hrac_as_hex), payload, verify=False) + + def ursula_from_rest_interface(self, address, port): + return requests.get("{}:{}/list_nodes".format(address, port), verify=False) # TODO: TLS-only. diff --git a/nkms/network/protocols.py b/nkms/network/protocols.py index f673b7a58..f951336bc 100644 --- a/nkms/network/protocols.py +++ b/nkms/network/protocols.py @@ -21,6 +21,9 @@ class NuCypherHashProtocol(KademliaProtocol): super().__init__(sourceNode, storage, ksize, *args, **kwargs) self.router = NuCypherRoutingTable(self, ksize, sourceNode) self.illegal_keys_seen = [] + # TODO: This is the wrong way to do this. See #227. + self.treasure_maps = {} + self.ursulas = {} def check_node_for_storage(self, node): try: @@ -91,6 +94,10 @@ class NuCypherHashProtocol(KademliaProtocol): if do_store: self.log.info("Storing k/v: {} / {}".format(key, value)) self.storage[key] = value + if value.startswith(bytes(constants.BYTESTRING_IS_URSULA_IFACE_INFO)): + self.ursulas[key] = value + if value.startswith(bytes(constants.BYTESTRING_IS_TREASURE_MAP)): + self.treasure_maps[key] = value return do_store diff --git a/nkms/network/server.py b/nkms/network/server.py index 239261587..c0fba81cf 100644 --- a/nkms/network/server.py +++ b/nkms/network/server.py @@ -5,7 +5,6 @@ from typing import ClassVar from apistar import http, Route, App from apistar.http import Response -from bytestring_splitter import BytestringSplitter from kademlia.crawling import NodeSpiderCrawl from kademlia.network import Server from kademlia.utils import digest @@ -13,7 +12,7 @@ from umbral import pre from umbral.fragments import KFrag from nkms.crypto.kits import UmbralMessageKit -from nkms.crypto.powers import EncryptingPower, SigningPower, CryptoPower +from nkms.crypto.powers import EncryptingPower, SigningPower from nkms.keystore.threading import ThreadedSession from nkms.network.capabilities import SeedOnly, ServerCapability from nkms.network.node import NuCypherNode @@ -117,6 +116,8 @@ class ProxyRESTServer(object): self.reencrypt_via_rest), Route('/public_keys', 'GET', self.get_signing_and_encrypting_public_keys), + Route('/list_nodes', 'GET', + self.list_all_active_nodes_about_which_we_know), Route('/consider_arrangement', 'POST', self.consider_arrangement), @@ -166,15 +167,20 @@ class ProxyRESTServer(object): return response + def list_all_active_nodes_about_which_we_know(self): + headers = {'Content-Type': 'application/octet-stream'} + ursulas_as_bytes = bytes().join(self.server.protocol.ursulas.values()) + signature = self.stamp(ursulas_as_bytes) + return Response(bytes(signature) + ursulas_as_bytes, headers=headers) + def consider_arrangement(self, request: http.Request): from nkms.policy.models import Arrangement - arrangement, deposit_as_bytes = BytestringSplitter(Arrangement)(request.body, return_remainder=True) - arrangement.deposit = deposit_as_bytes + arrangement = Arrangement.from_bytes(request.body) with ThreadedSession(self.db_engine) as session: self.datastore.add_policy_arrangement( arrangement.expiration.datetime(), - arrangement.deposit, + bytes(arrangement.deposit), hrac=arrangement.hrac.hex().encode(), alice_pubkey_sig=arrangement.alice.stamp, session=session, diff --git a/nkms/policy/models.py b/nkms/policy/models.py index 090380daf..7c053b004 100644 --- a/nkms/policy/models.py +++ b/nkms/policy/models.py @@ -24,7 +24,9 @@ class Arrangement(BlockchainArrangement): """ A Policy must be implemented by arrangements with n Ursulas. This class tracks the status of that implementation. """ - _EXPECTED_LENGTH = 99 + _EXPECTED_LENGTH = 106 + splitter = key_splitter + BytestringSplitter((bytes, KECCAK_DIGEST_LENGTH), + (bytes, 27), (bytes, 7)) def __init__(self, alice, hrac, expiration, deposit=None, ursula=None, kfrag=constants.UNKNOWN_KFRAG, alices_signature=None): @@ -66,10 +68,8 @@ class Arrangement(BlockchainArrangement): @classmethod def from_bytes(cls, arrangement_as_bytes): - arrangement_splitter = key_splitter + BytestringSplitter((bytes, KECCAK_DIGEST_LENGTH), - (bytes, 27)) - alice_pubkey_sig, hrac, expiration_bytes, deposit_bytes = arrangement_splitter( - arrangement_as_bytes, return_remainder=True) + # Still unclear how to arrive at the correct number of bytes to represent a deposit. See #148. + alice_pubkey_sig, hrac, expiration_bytes, deposit_bytes = cls.splitter(arrangement_as_bytes) expiration = maya.parse(expiration_bytes.decode()) alice = Alice.from_public_keys({SigningPower: alice_pubkey_sig}) return cls(alice=alice, hrac=hrac, expiration=expiration, deposit=int(deposit_bytes)) diff --git a/tests/network/test_network_actors.py b/tests/network/test_network_actors.py index 635c8b05a..9ae48958a 100644 --- a/tests/network/test_network_actors.py +++ b/tests/network/test_network_actors.py @@ -48,7 +48,7 @@ def test_vladimir_illegal_interface_key_does_not_propagate(ursulas): assert digest(illegal_key) in ursula.server.protocol.illegal_keys_seen -def test_alice_finds_ursula(alice, ursulas): +def test_alice_finds_ursula_via_dht(alice, ursulas): """ With the help of any Ursula, Alice can find a specific Ursula. """ @@ -63,6 +63,15 @@ def test_alice_finds_ursula(alice, ursulas): assert port == URSULA_PORT + ursula_index +def test_alice_finds_ursula_via_rest(alice, ursulas): + networky_stuff = MockNetworkyStuff(ursulas) + new_nodes = alice.learn_about_nodes(networky_stuff, address="https://localhost", port=ursulas[0].rest_port) + assert len(new_nodes) == len(ursulas) + + for ursula in ursulas: + assert ursula.stamp.as_umbral_pubkey() in new_nodes + + def test_alice_creates_policy_group_with_correct_hrac(idle_policy): """ Alice creates a PolicyGroup. It has the proper HRAC, unique per her, Bob, and the uri (resource_id). diff --git a/tests/utilities.py b/tests/utilities.py index 503ce859c..9973c9602 100644 --- a/tests/utilities.py +++ b/tests/utilities.py @@ -26,7 +26,7 @@ def make_ursulas(how_many_ursulas: int, ursula_starting_port: int) -> list: URSULAS = [] for _u in range(how_many_ursulas): port = ursula_starting_port + _u - _URSULA = Ursula(dht_port=port, dht_interface="127.0.0.1", db_name="test-{}".format(port)) + _URSULA = Ursula(dht_port=port, dht_interface="127.0.0.1", db_name="test-{}".format(port), rest_port=port+100) # TODO: Make ports unstupid and more clear. class MockDatastoreThreadPool(object): def callInThread(self, f, *args, **kwargs): @@ -84,3 +84,25 @@ class MockNetworkyStuff(NetworkyStuff): mock_client = TestClient(node.rest_app) return mock_client.get("http://localhost/treasure_map/{}".format(map_id.hex())) + def ursula_from_rest_interface(self, address, port): + for ursula in self.ursulas: + if ursula.rest_port == port: + rest_app = ursula.rest_app + break + else: + raise RuntimeError("Can't find that one - did you spin up the right test ursulas?") + mock_client = TestClient(ursula.rest_app) + response = mock_client.get("http://localhost/public_keys") + return response + + def get_nodes_via_rest(self, address, port): + for ursula in self.ursulas: + if ursula.rest_port == port: + rest_app = ursula.rest_app + break + else: + raise RuntimeError("Can't find that one - did you spin up the right test ursulas?") + mock_client = TestClient(ursula.rest_app) + response = mock_client.get("http://localhost/list_nodes") + return response + From f16f05e71ca14b509cd96883cd65f56dd178d31f Mon Sep 17 00:00:00 2001 From: jMyles Date: Sat, 14 Apr 2018 18:24:22 -0700 Subject: [PATCH 02/15] KMS now depends on bytestringSplitter branch kms-depend. --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index e6a144a9c..976ef792d 100644 --- a/Pipfile +++ b/Pipfile @@ -19,7 +19,7 @@ pyumbral = {git = "https://github.com/nucypher/pyumbral.git"} requests = "*" hendrix = {git = "https://github.com/hendrix/hendrix", ref = "tags/3.0.0rc1"} constantSorrow = {git = "https://github.com/nucypher/constantSorrow.git", ref = "kms-depend"} -bytestringSplitter = {git = "https://github.com/nucypher/byteStringSplitter.git", ref = "eaa1df2433362190f30bc6e400570f0331980ebb"} +bytestringSplitter = {git = "https://github.com/nucypher/byteStringSplitter.git", ref = "kms-depend"} appdirs = "*" populus = {git = "https://github.com/nucypher/Bropulus.git"} From 97f422436f1ba3319c6a3e04235990e3bf10a284 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:40:02 -0700 Subject: [PATCH 03/15] Attaching network middleware directly to Characters. --- nkms/characters.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nkms/characters.py b/nkms/characters.py index 41bc17636..25325bb5b 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -1,9 +1,7 @@ import asyncio from contextlib import suppress from logging import getLogger - import msgpack -import requests from collections import OrderedDict from kademlia.network import Server from kademlia.utils import digest @@ -11,6 +9,7 @@ from typing import Dict, ClassVar from typing import Union, List from bytestring_splitter import BytestringSplitter +from nkms.network.node import NetworkyStuff from umbral.keys import UmbralPublicKey from constant_sorrow import constants, default_constant_splitter @@ -22,7 +21,7 @@ from nkms.crypto.kits import UmbralMessageKit from nkms.crypto.powers import CryptoPower, SigningPower, EncryptingPower, DelegatingPower, NoSigningPower from nkms.crypto.signature import Signature, signature_splitter, SignatureStamp, StrangerStamp from nkms.network import blockchain_client -from nkms.network.protocols import dht_value_splitter +from nkms.network.protocols import dht_value_splitter, dht_with_hrac_splitter from nkms.network.server import NuCypherDHTServer, NuCypherSeedOnlyDHTServer, ProxyRESTServer @@ -38,7 +37,8 @@ class Character(object): address = "This is a fake address." # TODO: #192 def __init__(self, attach_server=True, crypto_power: CryptoPower=None, - crypto_power_ups=None, is_me=True, config: "KMSConfig"=None) -> None: + crypto_power_ups=None, is_me=True, network_middleware=None, + config: "KMSConfig"=None) -> None: """ :param attach_server: Whether to attach a Server when this Character is born. @@ -79,6 +79,7 @@ class Character(object): self._crypto_power = CryptoPower(self._default_crypto_powerups, generate_keys_if_needed=is_me) if is_me: + self.network_middleware = network_middleware or NetworkyStuff() try: self._stamp = SignatureStamp(self._crypto_power.power_ups(SigningPower).keypair) except NoSigningPower: @@ -87,6 +88,8 @@ class Character(object): if attach_server: self.attach_server() else: + if network_middleware is not None: + raise TypeError("Can't attach network middleware to a Character who isn't me. What are you even trying to do?") self._stamp = StrangerStamp(self._crypto_power.power_ups(SigningPower).keypair) def __eq__(self, other): From 9a0c7721cf165ebb8608f9e800bd2fe9e8c6eb12 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:40:25 -0700 Subject: [PATCH 04/15] New Character bootstrapping logic to allow Characters to connect to the network without the DHT. --- nkms/characters.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/nkms/characters.py b/nkms/characters.py index 25325bb5b..0fe786d34 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -281,26 +281,37 @@ class Character(object): power_up = self._crypto_power.power_ups(power_up_class) return power_up.public_key() - def learn_about_nodes(self, networky_stuff, address, port): + def learn_about_nodes(self, address, port): """ Sends a request to node_url to find out about known nodes. """ - response = networky_stuff.get_nodes_via_rest(address, port) + response = self.network_middleware.get_nodes_via_rest(address, port) signature, nodes = signature_splitter(response.content, return_remainder=True) # TODO: Although not treasure map-related, this has a whiff of #172. - ursula_interface_splitter = dht_value_splitter + BytestringSplitter((bytes, 15)) + ursula_interface_splitter = dht_value_splitter + BytestringSplitter((bytes, 17)) split_nodes = ursula_interface_splitter.repeat(nodes) new_nodes = {} - for node in split_nodes: - # Notice that we don't use "interface_hrac" - see #228. - header, sig, pubkey, interface_hrac, interface = node - if sig.verify(keccak_digest(interface), pubkey): - self.known_nodes[pubkey] = msgpack.loads(interface) - new_nodes[pubkey] = msgpack.loads(interface) - else: - self.log.warn("Discovered node with bad signature: {}".format(node)) + for node_meta in split_nodes: + header, sig, pubkey, interface_info = node_meta + if not pubkey in self.known_nodes: + if sig.verify(keccak_digest(interface_info), pubkey): + address, dht_port, rest_port = msgpack.loads(interface_info) + new_nodes[pubkey] = \ + Ursula.as_discovered_on_network( + rest_port=rest_port, + dht_port=dht_port, + ip_address=address.decode("utf-8"), + powers_and_keys=({SigningPower: pubkey}) + ) + else: + self.log.warn("Discovered node with bad signature: {}".format(node_meta)) return new_nodes + def network_bootstrap(self, node_list): + for node_addr, port in node_list: + new_nodes = self.learn_about_nodes(node_addr, port) + self.known_nodes.update(new_nodes) + class FakePolicyAgent: # TODO: #192 _token = "fake token" From ea722daa9bb55f4d37c36bff537bc846ab7f6bb4 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:41:18 -0700 Subject: [PATCH 05/15] Alice now publishes the TreasureMap as part of grant(). --- nkms/characters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nkms/characters.py b/nkms/characters.py index 0fe786d34..2b0b13e05 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -384,6 +384,7 @@ class Alice(Character, PolicyAuthor): policy.match_kfrags_to_found_ursulas(found_ursulas) # REST call happens here, as does population of TreasureMap. policy.enact(networky_stuff) + policy.publish_treasure_map(networky_stuff) return policy # Now with TreasureMap affixed! From bc31f325612ceee33c54391ccd1bd6ba21410d55 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:41:38 -0700 Subject: [PATCH 06/15] Bob can follow the TreasureMap without connecting to the DHT. --- nkms/characters.py | 75 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/nkms/characters.py b/nkms/characters.py index 2b0b13e05..4135723e2 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -400,33 +400,64 @@ class Bob(Character): from nkms.policy.models import WorkOrderHistory # Need a bigger strategy to avoid circulars. self._saved_work_orders = WorkOrderHistory() - def follow_treasure_map(self, hrac): - for ursula_interface_id in self.treasure_maps[hrac]: - if ursula_interface_id in self.known_nodes: - # If we already know about this Ursula, - # we needn't learn about it again. - continue + def follow_treasure_map(self, hrac, using_dht=False): - # TODO: perform this part concurrently. - value = self.server.get_now(ursula_interface_id) + treasure_map = self.treasure_maps[hrac] + number_of_known_treasure_ursulas = 0 + if not using_dht: + for ursula_interface_id in treasure_map: + pubkey = UmbralPublicKey.from_bytes(ursula_interface_id) + if pubkey in self.known_nodes: + number_of_known_treasure_ursulas += 1 - # TODO: Make this much prettier - header, signature, ursula_pubkey_sig, _hrac, ( - port, interface, ttl) = dht_value_splitter(value, msgpack_remainder=True) + newly_discovered_nodes = {} + nodes_to_check = iter(self.known_nodes.values()) - if header != constants.BYTESTRING_IS_URSULA_IFACE_INFO: - raise TypeError("Unknown DHT value. How did this get on the network?") + while number_of_known_treasure_ursulas < treasure_map.m: + try: + node_to_check = next(nodes_to_check) + except StopIteration: + raise self.NotEnoughUrsulas( + "Unable to follow the TreasureMap; we just don't know enough nodes to ask about this. Maybe try using the DHT instead.") - # TODO: If we're going to implement TTL, it will be here. - self.known_nodes[ursula_interface_id] = \ - Ursula.as_discovered_on_network( - dht_port=port, - dht_interface=interface, - powers_and_keys=({SigningPower: ursula_pubkey_sig}) - ) + new_nodes = self.learn_about_nodes(node_to_check.rest_address, + node_to_check.rest_port) + for new_node_pubkey in new_nodes.keys(): + if new_node_pubkey in treasure_map: + number_of_known_treasure_ursulas += 1 + newly_discovered_nodes.update(new_nodes) - def get_treasure_map(self, policy, networky_stuff, using_dht=False): - map_id = policy.treasure_map_dht_key() + self.known_nodes.update(newly_discovered_nodes) + return newly_discovered_nodes, number_of_known_treasure_ursulas + else: + for ursula_interface_id in self.treasure_maps[hrac]: + pubkey = UmbralPublicKey.from_bytes(ursula_interface_id) + if ursula_interface_id in self.known_nodes: + # If we already know about this Ursula, + # we needn't learn about it again. + continue + + if using_dht: + # TODO: perform this part concurrently. + value = self.server.get_now(ursula_interface_id) + + # TODO: Make this much prettier + header, signature, ursula_pubkey_sig, _hrac, ( + port, interface, ttl) = dht_value_splitter(value, msgpack_remainder=True) + + if header != constants.BYTESTRING_IS_URSULA_IFACE_INFO: + raise TypeError("Unknown DHT value. How did this get on the network?") + + # TODO: If we're going to implement TTL, it will be here. + self.known_nodes[ursula_interface_id] = \ + Ursula.as_discovered_on_network( + dht_port=port, + dht_interface=interface, + powers_and_keys=({SigningPower: ursula_pubkey_sig}) + ) + + def get_treasure_map(self, alice, hrac, using_dht=False): + map_id = keccak_digest(bytes(alice.stamp) + hrac) if using_dht: ursula_coro = self.server.get(map_id) From b1ed0a773672d985a468f2fb7a09ad7510e936f2 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:42:26 -0700 Subject: [PATCH 07/15] Keeping Ursula objects as known_nodes instead of tuples. Closes #239. Adding convenience logic for Bob. --- nkms/characters.py | 98 ++++++++++++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/nkms/characters.py b/nkms/characters.py index 4135723e2..8c8483bde 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -467,10 +467,11 @@ class Bob(Character): if not self.known_nodes: # TODO: Try to find more Ursulas on the blockchain. raise self.NotEnoughUrsulas - tmap_message_kit = self.get_treasure_map_from_known_ursulas(networky_stuff, map_id) + tmap_message_kit = self.get_treasure_map_from_known_ursulas(self.network_middleware, + map_id) verified, packed_node_list = self.verify_from( - policy.alice, tmap_message_kit, + alice, tmap_message_kit, decrypt=True ) @@ -478,8 +479,10 @@ class Bob(Character): return constants.NOT_FROM_ALICE else: from nkms.policy.models import TreasureMap - treasure_map = TreasureMap(msgpack.loads(packed_node_list)) - self.treasure_maps[policy.hrac()] = treasure_map + node_list = msgpack.loads(packed_node_list) + m = node_list.pop() + treasure_map = TreasureMap(m=m, ursula_interface_ids=node_list) + self.treasure_maps[hrac] = treasure_map return treasure_map def get_treasure_map_from_known_ursulas(self, networky_stuff, map_id): @@ -495,7 +498,7 @@ class Bob(Character): if response.status_code == 200 and response.content: # TODO: Make this prettier header, _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = \ - dht_value_splitter(response.content, return_remainder=True) + dht_with_hrac_splitter(response.content, return_remainder=True) tmap_messaage_kit = UmbralMessageKit.from_bytes(encrypted_treasure_map) return tmap_messaage_kit else: @@ -518,7 +521,7 @@ class Bob(Character): capsules)) for ursula_dht_key in treasure_map_to_use: - ursula = self.known_nodes[ursula_dht_key] + ursula = self.known_nodes[UmbralPublicKey.from_bytes(ursula_dht_key)] capsules_to_include = [] for capsule in capsules: @@ -551,20 +554,48 @@ class Bob(Character): def get_ursula(self, ursula_id): return self._ursulas[ursula_id] + def join_policy(self, alice, hrac, using_dht=False, node_list=None): + # TODO: unfuckify this + if node_list: + self.network_bootstrap(node_list) + self.get_treasure_map(alice, hrac, using_dht=using_dht) + self.follow_treasure_map(hrac, using_dht=using_dht) + + def retrieve(self, hrac, message_kit, data_source): + treasure_map = self.treasure_maps[hrac] + + # First, a quick sanity check to make sure we know about at least m nodes. + known_nodes_as_bytes = set([bytes(n) for n in self.known_nodes.keys()]) + intersection = treasure_map.ids.intersection(known_nodes_as_bytes) + + if len(intersection) < treasure_map.m: + raise RuntimeError("Not enough known nodes. Try following the TreasureMap again.") + + work_orders = self.generate_work_orders(hrac, message_kit.capsule) + for node_id in self.treasure_maps[hrac]: + node = self.known_nodes[UmbralPublicKey.from_bytes(node_id)] + cfrags = self.get_reencrypted_c_frags(self.network_middleware, work_orders[bytes(node.stamp)]) + message_kit.capsule.attach_cfrag(cfrags[0]) + verified, delivered_cleartext = self.verify_from(data_source, message_kit, decrypt=True) + + if verified: + return delivered_cleartext + else: + raise RuntimeError("Not verified - replace this with real message.") + class Ursula(Character, ProxyRESTServer): _server_class = NuCypherDHTServer _alice_class = Alice _default_crypto_powerups = [SigningPower, EncryptingPower] - def __init__(self, dht_port=None, dht_interface=None, dht_ttl=0, - rest_address=None, rest_port=None, db_name=None, + def __init__(self, dht_port=None, ip_address=None, dht_ttl=0, + rest_port=None, db_name=None, *args, **kwargs): self.dht_port = dht_port - self.dht_interface = dht_interface - self.dht_ttl = 0 + self.ip_address = ip_address self._work_orders = [] - ProxyRESTServer.__init__(self, rest_address, rest_port, db_name) + ProxyRESTServer.__init__(self, rest_port, db_name) super().__init__(*args, **kwargs) @property @@ -576,14 +607,12 @@ class Ursula(Character, ProxyRESTServer): return self._rest_app @classmethod - def as_discovered_on_network(cls, dht_port, dht_interface, - rest_address=None, rest_port=None, - powers_and_keys=()): + def as_discovered_on_network(cls, dht_port=None, ip_address=None, + rest_port=None, powers_and_keys=()): # TODO: We also need the encrypting public key here. ursula = cls.from_public_keys(powers_and_keys) ursula.dht_port = dht_port - ursula.dht_interface = dht_interface - ursula.rest_address = rest_address + ursula.ip_address = ip_address ursula.rest_port = rest_port return ursula @@ -593,13 +622,13 @@ class Ursula(Character, ProxyRESTServer): if not response.status_code == 200: raise RuntimeError("Got a bad response: {}".format(response)) - key_splitter = RepeatingBytestringSplitter( + key_splitter = BytestringSplitter( (UmbralPublicKey, PUBLIC_KEY_LENGTH)) - signing_key, encrypting_key = key_splitter(response.content) + signing_key, encrypting_key = key_splitter.repeat(response.content) stranger_ursula_from_public_keys = cls.from_public_keys( {SigningPower: signing_key, EncryptingPower: encrypting_key}, - rest_address=address, + ip_address=address, rest_port=port ) @@ -615,32 +644,25 @@ class Ursula(Character, ProxyRESTServer): super().attach_server(ksize, alpha, id, storage) self.attach_rest_server(db_name=self.db_name) - def listen(self): - return self.server.listen(self.dht_port, self.dht_interface) + def dht_listen(self): + return self.server.listen(self.dht_port, self.ip_address) - def dht_interface_info(self): - return self.dht_port, self.dht_interface, self.dht_ttl + def interface_information(self): + return msgpack.dumps((self.ip_address, + self.dht_port, + self.rest_port)) - def interface_dht_key(self): - return bytes(self.stamp) - # return self.InterfaceDHTKey(self.stamp, self.interface_hrac()) - - def interface_dht_value(self): - signature = self.stamp(self.interface_hrac()) - return ( - constants.BYTESTRING_IS_URSULA_IFACE_INFO + signature + self.stamp + self.interface_hrac() - + msgpack.dumps(self.dht_interface_info()) - ) - - def interface_hrac(self): - return keccak_digest(msgpack.dumps(self.dht_interface_info())) + def interface_info_with_metadata(self): + interface_info = self.interface_information() + signature = self.stamp(keccak_digest(interface_info)) + return constants.BYTESTRING_IS_URSULA_IFACE_INFO + signature + self.stamp + interface_info def publish_dht_information(self): if not self.dht_port and self.dht_interface: raise RuntimeError("Must listen before publishing interface information.") - dht_key = self.interface_dht_key() - value = self.interface_dht_value() + dht_key = bytes(self.stamp) + value = self.interface_info_with_metadata() setter = self.server.set(key=dht_key, value=value) blockchain_client._ursulas_on_blockchain.append(dht_key) loop = asyncio.get_event_loop() From d32f290d6b8e090936f31cd60aa6fec3d63a5886 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:44:26 -0700 Subject: [PATCH 08/15] DataSource now automatically generates a signer - not sure how we want to generate this keypair. See #241. --- nkms/data_sources.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nkms/data_sources.py b/nkms/data_sources.py index 0af3fda24..d57c3c855 100644 --- a/nkms/data_sources.py +++ b/nkms/data_sources.py @@ -1,10 +1,14 @@ from nkms.crypto.api import encrypt_and_sign - +from nkms.crypto.signature import SignatureStamp +from nkms.keystore.keypairs import SigningKeypair +from constant_sorrow.constants import NO_SIGNING_POWER class DataSource: - def __init__(self, policy_pubkey_enc, signer): + def __init__(self, policy_pubkey_enc, signer=NO_SIGNING_POWER): self._policy_pubkey_enc = policy_pubkey_enc + if signer is NO_SIGNING_POWER: + signer = SignatureStamp(SigningKeypair()) # TODO: Generate signing key properly. #241 self.stamp = signer def encapsulate_single_message(self, message): From 06c1c6b06b30356483247fc23baf55c18868e115 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:45:22 -0700 Subject: [PATCH 09/15] Updating tests and splitters. --- nkms/keystore/keystore.py | 3 +++ nkms/network/node.py | 19 ++++++++++++++----- nkms/network/protocols.py | 18 ++++++++++++------ nkms/network/server.py | 35 ++++++++++++++++++++--------------- 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/nkms/keystore/keystore.py b/nkms/keystore/keystore.py index b181e7659..fb3b12832 100644 --- a/nkms/keystore/keystore.py +++ b/nkms/keystore/keystore.py @@ -138,6 +138,9 @@ class KeyStore(object): policy_arrangement = session.query(PolicyArrangement).filter_by(hrac=hrac_as_hex.encode()).first() + if policy_arrangement is None: + raise NotFound("Can't attach a kfrag to non-existent Arrangement with hrac {}".format(hrac_as_hex)) + if policy_arrangement.alice_pubkey_sig.key_data != alice.stamp: raise alice.SuspiciousActivity diff --git a/nkms/network/node.py b/nkms/network/node.py index aa0faea85..23d5e4e5c 100644 --- a/nkms/network/node.py +++ b/nkms/network/node.py @@ -46,19 +46,28 @@ class NetworkyStuff(object): return NotImplemented def get_treasure_map_from_node(self, node, map_id): - response = requests.get("{}/treasure_map/{}".format(node.rest_url(), map_id.hex()), verify=False) + port = node.rest_port + address = node.ip_address + endpoint = "https://{}:{}/treasure_map/{}".format(address, port, map_id.hex()) + response = requests.get(endpoint, verify=False) return response def push_treasure_map_to_node(self, node, map_id, map_payload): - response = requests.post("{}/treasure_map/{}".format(node.rest_url(), map_id.hex()), - data=map_payload, verify=False) + port = node.rest_port + address = node.ip_address + endpoint = "https://{}:{}/treasure_map/{}".format(address, port, map_id.hex()) + response = requests.post(endpoint, data=map_payload, verify=False) return response def send_work_order_payload_to_ursula(self, work_order): payload = work_order.payload() hrac_as_hex = work_order.kfrag_hrac.hex() - return requests.post('{}/kFrag/{}/reencrypt'.format(work_order.ursula.rest_url(), hrac_as_hex), + return requests.post('https://{}/kFrag/{}/reencrypt'.format(work_order.ursula.rest_url(), hrac_as_hex), payload, verify=False) def ursula_from_rest_interface(self, address, port): - return requests.get("{}:{}/list_nodes".format(address, port), verify=False) # TODO: TLS-only. + return requests.get("https://{}:{}/public_keys".format(address, port), verify=False) # TODO: TLS-only. + + def get_nodes_via_rest(self, address, port): + response = requests.get("https://{}:{}/list_nodes".format(address, port), verify=False) # TODO: TLS-only. + return response diff --git a/nkms/network/protocols.py b/nkms/network/protocols.py index f951336bc..76bfd8dd0 100644 --- a/nkms/network/protocols.py +++ b/nkms/network/protocols.py @@ -11,9 +11,8 @@ from nkms.network.node import NuCypherNode from nkms.network.routing import NuCypherRoutingTable from umbral.keys import UmbralPublicKey -dht_value_splitter = default_constant_splitter + BytestringSplitter(Signature, - (UmbralPublicKey, PUBLIC_KEY_LENGTH), - (bytes, KECCAK_DIGEST_LENGTH)) +dht_value_splitter = default_constant_splitter + BytestringSplitter(Signature, (UmbralPublicKey, PUBLIC_KEY_LENGTH)) +dht_with_hrac_splitter = dht_value_splitter + BytestringSplitter((bytes, KECCAK_DIGEST_LENGTH)) class NuCypherHashProtocol(KademliaProtocol): @@ -78,9 +77,16 @@ class NuCypherHashProtocol(KademliaProtocol): self.log.debug("got a store request from %s" % str(sender)) # TODO: Why is this logic here? This is madness. See #172. - if value.startswith(bytes(constants.BYTESTRING_IS_URSULA_IFACE_INFO)) or value.startswith( - bytes(constants.BYTESTRING_IS_TREASURE_MAP)): - header, signature, sender_pubkey_sig, hrac, message = dht_value_splitter( + if value.startswith(bytes(constants.BYTESTRING_IS_URSULA_IFACE_INFO)): + header, signature, sender_pubkey_sig, message = dht_value_splitter( + value, return_remainder=True) + + # TODO: TTL? + hrac = keccak_digest(message) + do_store = self.determine_legality_of_dht_key(signature, sender_pubkey_sig, message, + hrac, key, value) + elif value.startswith(bytes(constants.BYTESTRING_IS_TREASURE_MAP)): + header, signature, sender_pubkey_sig, hrac, message = dht_with_hrac_splitter( value, return_remainder=True) # TODO: TTL? diff --git a/nkms/network/server.py b/nkms/network/server.py index c0fba81cf..852a87456 100644 --- a/nkms/network/server.py +++ b/nkms/network/server.py @@ -17,7 +17,7 @@ from nkms.keystore.threading import ThreadedSession from nkms.network.capabilities import SeedOnly, ServerCapability from nkms.network.node import NuCypherNode from nkms.network.protocols import NuCypherSeedOnlyProtocol, NuCypherHashProtocol, \ - dht_value_splitter + dht_value_splitter, dht_with_hrac_splitter from nkms.network.storage import SeedOnlyStorage @@ -95,8 +95,7 @@ class NuCypherSeedOnlyDHTServer(NuCypherDHTServer): class ProxyRESTServer(object): - def __init__(self, rest_address, rest_port, db_name): - self.rest_address = rest_address + def __init__(self, rest_port, db_name): self.rest_port = rest_port self.db_name = db_name self._rest_app = None @@ -146,14 +145,12 @@ class ProxyRESTServer(object): self.db_engine = engine def rest_url(self): - return "{}:{}".format(self.rest_address, self.rest_port) + return "{}:{}".format(self.ip_address, self.rest_port) - # """ + + ##################################### # Actual REST Endpoints and utilities - # """ - # def find_ursulas_by_ids(self, request: http.Request): - # - # + ##################################### def get_signing_and_encrypting_public_keys(self): """ @@ -170,6 +167,7 @@ class ProxyRESTServer(object): def list_all_active_nodes_about_which_we_know(self): headers = {'Content-Type': 'application/octet-stream'} ursulas_as_bytes = bytes().join(self.server.protocol.ursulas.values()) + ursulas_as_bytes += self.interface_info_with_metadata() signature = self.stamp(ursulas_as_bytes) return Response(bytes(signature) + ursulas_as_bytes, headers=headers) @@ -178,7 +176,7 @@ class ProxyRESTServer(object): arrangement = Arrangement.from_bytes(request.body) with ThreadedSession(self.db_engine) as session: - self.datastore.add_policy_arrangement( + new_policyarrangement = self.datastore.add_policy_arrangement( arrangement.expiration.datetime(), bytes(arrangement.deposit), hrac=arrangement.hrac.hex().encode(), @@ -189,6 +187,7 @@ class ProxyRESTServer(object): # to decide if this Arrangement is worth accepting. headers = {'Content-Type': 'application/octet-stream'} + # TODO: Make this a legit response #234. return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers) def set_policy(self, hrac_as_hex, request: http.Request): @@ -254,25 +253,31 @@ class ProxyRESTServer(object): def provide_treasure_map(self, treasure_map_id_as_hex): # For now, grab the TreasureMap for the DHT storage. Soon, no do that. #TODO! treasure_map_id = binascii.unhexlify(treasure_map_id_as_hex) - treasure_map_bytes = self.server.storage.get(digest(treasure_map_id)) headers = {'Content-Type': 'application/octet-stream'} - return Response(content=treasure_map_bytes, headers=headers) + try: + treasure_map_bytes = self.server.storage[digest(treasure_map_id)] + response = Response(content=treasure_map_bytes, headers=headers) + except KeyError: + response = Response("No Treasure Map with ID {}".format(treasure_map_id), + status_code=404, headers=headers) + + return response def receive_treasure_map(self, treasure_map_id_as_hex, request: http.Request): # TODO: This function is the epitome of #172. treasure_map_id = binascii.unhexlify(treasure_map_id_as_hex) header, signature_for_ursula, pubkey_sig_alice, hrac, tmap_message_kit = \ - dht_value_splitter(request.body, return_remainder=True) + dht_with_hrac_splitter(request.body, return_remainder=True) # TODO: This next line is possibly the worst in the entire codebase at the moment. #172. # Also TODO: TTL? do_store = self.server.protocol.determine_legality_of_dht_key( signature_for_ursula, pubkey_sig_alice, tmap_message_kit, hrac, digest(treasure_map_id), request.body) if do_store: - # TODO: Stop storing things in the protocol storage. Do this better. - # TODO: Propagate to other nodes. + # TODO: Stop storing things in the protocol storage. Do this better. #227 + # TODO: Propagate to other nodes. #235 self.server.protocol.storage[digest(treasure_map_id)] = request.body return # TODO: Proper response here. else: From 4e86386caf2397e854cf4282be7a15b00c69cc85 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:45:41 -0700 Subject: [PATCH 10/15] TreasureMap touchups, including the value of m. See #238. --- nkms/policy/models.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/nkms/policy/models.py b/nkms/policy/models.py index 7c053b004..2649f4ff7 100644 --- a/nkms/policy/models.py +++ b/nkms/policy/models.py @@ -114,18 +114,18 @@ class Policy(object): """ _ursula = None - def __init__(self, alice, bob=None, kfrags=(constants.UNKNOWN_KFRAG,), uri=None, m=None, alices_signature=constants.NOT_SIGNED): + def __init__(self, alice, bob=None, kfrags=(constants.UNKNOWN_KFRAG,), + label=None, m=None, alices_signature=constants.NOT_SIGNED): """ :param kfrags: A list of KFrags to distribute per this Policy. - :param uri: The identity of the resource to which Bob is granted access. + :param label: The identity of the resource to which Bob is granted access. """ self.alice = alice self.bob = bob self.kfrags = kfrags - self.uri = uri - self.m = m - self.treasure_map = TreasureMap() + self.uri = label + self.treasure_map = TreasureMap(m=m) self._accepted_arrangements = OrderedDict() self.alices_signature = alices_signature @@ -195,7 +195,7 @@ class Policy(object): """ return keccak_digest(bytes(self.alice.stamp) + self.hrac()) - def publish_treasure_map(self, networky_stuff=None, use_dht=True): + def publish_treasure_map(self, networky_stuff=None, use_dht=False): if networky_stuff is None and use_dht is False: raise ValueError("Can't engage the REST swarm without networky stuff.") tmap_message_kit, signature_for_bob = self.alice.encrypt_for( @@ -210,10 +210,13 @@ class Policy(object): map_id = self.treasure_map_dht_key() if use_dht: + # Instead of self.alice, let's say self.author. See #230. setter = self.alice.server.set(map_id, constants.BYTESTRING_IS_TREASURE_MAP + map_payload) event_loop = asyncio.get_event_loop() event_loop.run_until_complete(setter) else: + if not self.alice.known_nodes: + raise RuntimeError("Alice hasn't learned of any nodes. Thus, she can't push the TreasureMap.") for node in self.alice.known_nodes.values(): response = networky_stuff.push_treasure_map_to_node(node, map_id, constants.BYTESTRING_IS_TREASURE_MAP + map_payload) # TODO: Do something here based on success or failure @@ -274,15 +277,20 @@ class Policy(object): def public_key(self): return self.alice.public_key(DelegatingPower) + class TreasureMap(object): - def __init__(self, ursula_interface_ids=None): - self.ids = ursula_interface_ids or [] + def __init__(self, m, ursula_interface_ids=None): + self.m = m + self.ids = set(ursula_interface_ids or set()) def packed_payload(self): - return msgpack.dumps([bytes(ursula_id) for ursula_id in self.ids]) + return msgpack.dumps(self.nodes_as_bytes() + [self.m]) + + def nodes_as_bytes(self): + return [bytes(ursula_id) for ursula_id in self.ids] def add_ursula(self, ursula): - self.ids.append(ursula.interface_dht_key()) + self.ids.add(bytes(ursula.stamp)) def __eq__(self, other): return self.ids == other.ids @@ -319,7 +327,7 @@ class WorkOrder(object): @classmethod def construct_by_bob(cls, kfrag_hrac, capsules, ursula, bob): - receipt_bytes = b"wo:" + ursula.interface_dht_key() # TODO: represent the capsules as bytes and hash them as part of the receipt, ie + keccak_digest(b"".join(capsules)) - See #137 + receipt_bytes = b"wo:" + ursula.interface_information() # TODO: represent the capsules as bytes and hash them as part of the receipt, ie + keccak_digest(b"".join(capsules)) - See #137 receipt_signature = bob.stamp(receipt_bytes) return cls(bob, kfrag_hrac, capsules, receipt_bytes, receipt_signature, ursula) From 3f55470870b756d9fa40b66bbf143f6ad0d333e9 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:46:13 -0700 Subject: [PATCH 11/15] Tests for new Bob and TreasureMap logic. --- tests/characters/test_bob_handles_frags.py | 68 ++++++++++++++++++---- tests/fixtures.py | 21 +++---- tests/network/test_network_actors.py | 29 +++++---- tests/utilities.py | 41 ++++++++++--- 4 files changed, 112 insertions(+), 47 deletions(-) diff --git a/tests/characters/test_bob_handles_frags.py b/tests/characters/test_bob_handles_frags.py index 461a9cc73..f396013dd 100644 --- a/tests/characters/test_bob_handles_frags.py +++ b/tests/characters/test_bob_handles_frags.py @@ -5,24 +5,68 @@ from umbral import pre from umbral.fragments import KFrag -def test_bob_can_follow_treasure_map(enacted_policy, ursulas, alice, bob): - """ - Upon receiving a TreasureMap, Bob populates his list of Ursulas with the correct number. - """ +def test_bob_cannot_follow_the_treasure_map_in_isolation(enacted_policy, bob): - # Simulate Bob finding a TreasureMap on the DHT. - # A test to show that Bob can do this can be found in test_network_actors. + # Assume for the moment that Bob has already received a TreasureMap, perhaps via a side channel. hrac, treasure_map = enacted_policy.hrac(), enacted_policy.treasure_map bob.treasure_maps[hrac] = treasure_map # Bob knows of no Ursulas. assert len(bob.known_nodes) == 0 - # ...until he follows the TreasureMap. - bob.follow_treasure_map(hrac) + # He can't successfully follow the TreasureMap until he learns of a node to ask. + with pytest.raises(bob.NotEnoughUrsulas): + bob.follow_treasure_map(hrac) - # Now he knows of all the Ursulas. - assert len(bob.known_nodes) == len(treasure_map) + +@pytest.mark.usefixtures("treasure_map_is_set_on_dht") +def test_bob_can_follow_treasure_map(enacted_policy, ursulas, bob, alice): + """ + Similar to above, but this time, we'll show that if Bob can connect to a single node, he can + learn enough to follow the TreasureMap. + + Also, we'll get the TreasureMap from the hrac alone (ie, not via a side channel). + """ + hrac = enacted_policy.hrac() + + # Bob knows of no Ursulas. + assert len(bob.known_nodes) == 0 + + # Now Bob will ask just a single Ursula to tell him of the nodes about which she's aware. + bob.network_bootstrap([("127.0.0.1", ursulas[0].rest_port)]) + + # Success! This Ursula knew about all the nodes on the network! + assert len(bob.known_nodes) == len(ursulas) + + # Now, Bob can get the TreasureMap all by himself, and doesn't need a side channel. + bob.get_treasure_map(alice, hrac) + newly_discovered, total_known = bob.follow_treasure_map(hrac) + + # He finds that he didn't need to discover any new nodes... + assert len(newly_discovered) == 0 + + # ...because he already knew of all the Ursulas on the map. + assert total_known == len(enacted_policy.treasure_map) + + +def test_bob_can_follow_treasure_map_even_if_he_only_knows_of_one_node(enacted_policy, ursulas, bob): + # Again, let's assume that he received the TreasureMap via a side channel. + hrac, treasure_map = enacted_policy.hrac(), enacted_policy.treasure_map + bob.treasure_maps[hrac] = treasure_map + + # Now, let's create a scenario in which Bob knows of only one node. + k, v = bob.known_nodes.popitem() + bob.known_nodes = {k: v} + assert len(bob.known_nodes) == 1 + + # This time, when he follows the TreasureMap... + newly_discovered, total_known = bob.follow_treasure_map(hrac) + + # The newly discovered nodes are all those in the TreasureMap except the one about which he already knew. + assert len(newly_discovered) == len(treasure_map) - 1 + + # ...and his total known now matches the length of the TreasureMap. + assert total_known == len(treasure_map) def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice, bob, ursulas, @@ -145,13 +189,13 @@ def test_bob_gathers_and_combines(enacted_policy, bob, ursulas, capsule_side_cha assert len(bob._saved_work_orders) == 2 # ...but the policy requires us to collect more cfrags. - assert len(bob._saved_work_orders) < enacted_policy.m + assert len(bob._saved_work_orders) < enacted_policy.treasure_map.m # Bob can't decrypt yet with just two CFrags. He needs to gather at least m. with pytest.raises(pre.GenericUmbralError): bob.decrypt(the_message_kit) - number_left_to_collect = enacted_policy.m - len(bob._saved_work_orders) + number_left_to_collect = enacted_policy.treasure_map.m - len(bob._saved_work_orders) new_work_orders = bob.generate_work_orders(enacted_policy.hrac(), the_message_kit.capsule, diff --git a/tests/fixtures.py b/tests/fixtures.py index 6a5470037..07ce1a866 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -5,12 +5,7 @@ import maya import pytest from constant_sorrow import constants from sqlalchemy.engine import create_engine -from umbral import pre - from nkms.characters import Alice, Bob - -from nkms.crypto.kits import MessageKit -from nkms.crypto.powers import EncryptingPower from nkms.keystore import keystore from nkms.keystore.db import Base @@ -19,7 +14,6 @@ from nkms.data_sources import DataSource from nkms.keystore.keypairs import SigningKeypair from nkms.network import blockchain_client -from nkms.policy.models import Arrangement from tests.utilities import NUMBER_OF_URSULAS_IN_NETWORK, MockNetworkyStuff, make_ursulas, \ URSULA_PORT, EVENT_LOOP @@ -46,7 +40,6 @@ def enacted_policy(idle_policy, ursulas): # Alice has a policy in mind and knows of enough qualifies Ursulas; she crafts an offer for them. deposit = constants.NON_PAYMENT contract_end_datetime = maya.now() + datetime.timedelta(days=5) - # contract = Contract(idle_policy.n, deposit, contract_end_datetime) networky_stuff = MockNetworkyStuff(ursulas) found_ursulas = idle_policy.find_ursulas(networky_stuff, deposit, expiration=contract_end_datetime) @@ -58,18 +51,17 @@ def enacted_policy(idle_policy, ursulas): @pytest.fixture(scope="module") def alice(ursulas): - ALICE = Alice() + ALICE = Alice(network_middleware=MockNetworkyStuff(ursulas)) ALICE.server.listen(8471) ALICE.__resource_id = b"some_resource_id" EVENT_LOOP.run_until_complete(ALICE.server.bootstrap([("127.0.0.1", u.dht_port) for u in ursulas])) + ALICE.network_bootstrap([("127.0.0.1", u.rest_port) for u in ursulas]) return ALICE @pytest.fixture(scope="module") -def bob(): - BOB = Bob() - BOB.server.listen(8475) - EVENT_LOOP.run_until_complete(BOB.server.bootstrap([("127.0.0.1", URSULA_PORT)])) +def bob(ursulas): + BOB = Bob(network_middleware=MockNetworkyStuff(ursulas)) return BOB @@ -85,8 +77,9 @@ def ursulas(): @pytest.fixture(scope="module") -def treasure_map_is_set_on_dht(enacted_policy): - enacted_policy.publish_treasure_map() +def treasure_map_is_set_on_dht(enacted_policy, ursulas): + networky_stuff = MockNetworkyStuff(ursulas) + enacted_policy.publish_treasure_map(networky_stuff, use_dht=True) @pytest.fixture(scope="module") diff --git a/tests/network/test_network_actors.py b/tests/network/test_network_actors.py index 9ae48958a..bd5f688ac 100644 --- a/tests/network/test_network_actors.py +++ b/tests/network/test_network_actors.py @@ -7,7 +7,7 @@ from kademlia.utils import digest from nkms.crypto.api import keccak_digest from nkms.crypto.kits import UmbralMessageKit from nkms.network import blockchain_client -from nkms.network.protocols import dht_value_splitter +from nkms.network.protocols import dht_value_splitter, dht_with_hrac_splitter from tests.utilities import MockNetworkyStuff, EVENT_LOOP, URSULA_PORT, NUMBER_OF_URSULAS_IN_NETWORK @@ -36,7 +36,7 @@ def test_vladimir_illegal_interface_key_does_not_propagate(ursulas): assert ursula.server.protocol.illegal_keys_seen == [] # Vladimir does almost everything right.... - value = vladimir.interface_dht_value() + value = vladimir.interface_info_with_metadata() # Except he sets an illegal key for his interface. illegal_key = b"Not allowed to set arbitrary key for this." @@ -55,17 +55,21 @@ def test_alice_finds_ursula_via_dht(alice, ursulas): ursula_index = 1 all_ursulas = blockchain_client._ursulas_on_blockchain value = alice.server.get_now(all_ursulas[ursula_index]) - header, _signature, _ursula_pubkey_sig, _hrac, interface_info = dht_value_splitter(value, + header, _signature, _ursula_pubkey_sig, interface_info = dht_value_splitter(value, return_remainder=True) assert header == constants.BYTESTRING_IS_URSULA_IFACE_INFO - port = msgpack.loads(interface_info)[0] + port = msgpack.loads(interface_info)[1] assert port == URSULA_PORT + ursula_index def test_alice_finds_ursula_via_rest(alice, ursulas): networky_stuff = MockNetworkyStuff(ursulas) - new_nodes = alice.learn_about_nodes(networky_stuff, address="https://localhost", port=ursulas[0].rest_port) + + # Imagine alice knows of nobody. + alice.known_nodes = {} + + new_nodes = alice.learn_about_nodes(address="https://localhost", port=ursulas[0].rest_port) assert len(new_nodes) == len(ursulas) for ursula in ursulas: @@ -87,7 +91,8 @@ def test_alice_sets_treasure_map_on_network(enacted_policy, ursulas): """ Having enacted all the policies of a PolicyGroup, Alice creates a TreasureMap and sends it to Ursula via the DHT. """ - _, packed_encrypted_treasure_map, _, _ = enacted_policy.publish_treasure_map() + networky_stuff = MockNetworkyStuff(ursulas) + _, packed_encrypted_treasure_map, _, _ = enacted_policy.publish_treasure_map(networky_stuff=networky_stuff, use_dht=True) treasure_map_as_set_on_network = ursulas[0].server.storage[ digest(enacted_policy.treasure_map_dht_key())] @@ -120,7 +125,7 @@ def test_treasure_map_stored_by_ursula_is_the_correct_one_for_bob(alice, bob, ur treasure_map_as_set_on_network = ursulas[0].server.storage[ digest(enacted_policy.treasure_map_dht_key())] - header, _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = dht_value_splitter( + header, _signature_for_ursula, pubkey_sig_alice, hrac, encrypted_treasure_map = dht_with_hrac_splitter( treasure_map_as_set_on_network, return_remainder=True) assert header == constants.BYTESTRING_IS_TREASURE_MAP @@ -149,13 +154,13 @@ def test_bob_can_retreive_the_treasure_map_and_decrypt_it(enacted_policy, ursula # If Bob doesn't know about any Ursulas, he can't find the TreasureMap via the REST swarm: with pytest.raises(bob.NotEnoughUrsulas): - treasure_map_from_wire = bob.get_treasure_map(enacted_policy, networky_stuff) + treasure_map_from_wire = bob.get_treasure_map(enacted_policy.alice, enacted_policy.hrac()) # Let's imagine he has learned about some - say, from the blockchain. - bob.known_nodes = {u.interface_dht_key(): u for u in ursulas} + bob.known_nodes = {u.interface_info_with_metadata(): u for u in ursulas} # Now try. - treasure_map_from_wire = bob.get_treasure_map(enacted_policy, networky_stuff) + treasure_map_from_wire = bob.get_treasure_map(enacted_policy.alice, enacted_policy.hrac()) assert enacted_policy.treasure_map == treasure_map_from_wire @@ -167,10 +172,10 @@ def test_treaure_map_is_legit(enacted_policy): alice = enacted_policy.alice for ursula_interface_id in enacted_policy.treasure_map: value = alice.server.get_now(ursula_interface_id) - header, signature, ursula_pubkey_sig, hrac, interface_info = dht_value_splitter(value, + header, signature, ursula_pubkey_sig, interface_info = dht_value_splitter(value, return_remainder=True) assert header == constants.BYTESTRING_IS_URSULA_IFACE_INFO - port = msgpack.loads(interface_info)[0] + port = msgpack.loads(interface_info)[1] legal_ports = range(NUMBER_OF_URSULAS_IN_NETWORK, NUMBER_OF_URSULAS_IN_NETWORK + URSULA_PORT) assert port in legal_ports diff --git a/tests/utilities.py b/tests/utilities.py index 9973c9602..7bdb91064 100644 --- a/tests/utilities.py +++ b/tests/utilities.py @@ -26,14 +26,14 @@ def make_ursulas(how_many_ursulas: int, ursula_starting_port: int) -> list: URSULAS = [] for _u in range(how_many_ursulas): port = ursula_starting_port + _u - _URSULA = Ursula(dht_port=port, dht_interface="127.0.0.1", db_name="test-{}".format(port), rest_port=port+100) # TODO: Make ports unstupid and more clear. + _URSULA = Ursula(dht_port=port, ip_address="127.0.0.1", db_name="test-{}".format(port), rest_port=port+100) # TODO: Make ports unstupid and more clear. class MockDatastoreThreadPool(object): def callInThread(self, f, *args, **kwargs): return f(*args, **kwargs) _URSULA.datastore_threadpool = MockDatastoreThreadPool() - _URSULA.listen() + _URSULA.dht_listen() URSULAS.append(_URSULA) @@ -53,8 +53,9 @@ class MockArrangementResponse(ArrangementResponse): class MockNetworkyStuff(NetworkyStuff): + def __init__(self, ursulas): - self._ursulas = {u.interface_dht_key(): u for u in ursulas} + self._ursulas = {bytes(u.stamp): u for u in ursulas} self.ursulas = iter(ursulas) def go_live_with_policy(self, ursula, policy_offer): @@ -72,7 +73,7 @@ class MockNetworkyStuff(NetworkyStuff): def enact_policy(self, ursula, hrac, payload): mock_client = TestClient(ursula.rest_app) response = mock_client.post('http://localhost/kFrag/{}'.format(hrac.hex()), payload) - return True, ursula.interface_dht_key() + return True, ursula.stamp.as_umbral_pubkey() def send_work_order_payload_to_ursula(self, work_order): mock_client = TestClient(work_order.ursula.rest_app) @@ -81,28 +82,50 @@ class MockNetworkyStuff(NetworkyStuff): return mock_client.post('http://localhost/kFrag/{}/reencrypt'.format(hrac_as_hex), payload) def get_treasure_map_from_node(self, node, map_id): - mock_client = TestClient(node.rest_app) + for ursula in self._ursulas.values(): + if ursula.rest_port == node.rest_port: + rest_app = ursula.rest_app + break + else: + raise RuntimeError( + "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) + mock_client = TestClient(ursula.rest_app) return mock_client.get("http://localhost/treasure_map/{}".format(map_id.hex())) def ursula_from_rest_interface(self, address, port): - for ursula in self.ursulas: + for ursula in self._ursulas.values(): if ursula.rest_port == port: rest_app = ursula.rest_app break else: - raise RuntimeError("Can't find that one - did you spin up the right test ursulas?") + raise RuntimeError( + "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) mock_client = TestClient(ursula.rest_app) response = mock_client.get("http://localhost/public_keys") return response def get_nodes_via_rest(self, address, port): - for ursula in self.ursulas: + for ursula in self._ursulas.values(): if ursula.rest_port == port: rest_app = ursula.rest_app break else: - raise RuntimeError("Can't find that one - did you spin up the right test ursulas?") + raise RuntimeError("Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) mock_client = TestClient(ursula.rest_app) response = mock_client.get("http://localhost/list_nodes") return response + def push_treasure_map_to_node(self, node, map_id, map_payload): + port = node.rest_port + for ursula in self._ursulas.values(): + if ursula.rest_port == port: + rest_app = ursula.rest_app + break + else: + raise RuntimeError( + "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) + mock_client = TestClient(ursula.rest_app) + response = mock_client.post("http://localhost/treasure_map/{}".format(map_id.hex()), + data=map_payload, verify=False) + return response + From 4e6a0409ce65bf03d67976b83ddea1c06e577f58 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 01:46:23 -0700 Subject: [PATCH 12/15] Finnegan's Wake demo updated: * No longer uses the DHT for Alice and Bob * Uses new higher-level methods (and is much shorter). --- .../alicebob-grant-and-line-by-line-PRE.py | 49 +++++++++---------- ..._ursula_with_rest_and_dht_but_no_mining.py | 8 +-- examples/sandbox_resources.py | 32 ++++++++++++ 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/examples/alicebob-grant-and-line-by-line-PRE.py b/examples/alicebob-grant-and-line-by-line-PRE.py index b9c906919..03b6b5b2b 100644 --- a/examples/alicebob-grant-and-line-by-line-PRE.py +++ b/examples/alicebob-grant-and-line-by-line-PRE.py @@ -8,38 +8,39 @@ import sys from examples.sandbox_resources import SandboxNetworkyStuff from nkms.characters import Alice, Bob, Ursula -from nkms.crypto.kits import MessageKit -from nkms.crypto.powers import SigningPower, EncryptingPower +from nkms.crypto.api import keccak_digest +from nkms.data_sources import DataSource from nkms.network.node import NetworkyStuff -from umbral import pre +import maya + +# Some basic setup. ALICE = Alice() BOB = Bob() -URSULA = Ursula.from_rest_url(address="https://localhost", port="3550") +URSULA = Ursula.from_rest_url(NetworkyStuff(), address="localhost", port=3601) +network_middleware = SandboxNetworkyStuff([URSULA]) -networky_stuff = SandboxNetworkyStuff() - -policy_end_datetime = datetime.datetime.now() + datetime.timedelta(days=5) +# Here are our Policy details. +policy_end_datetime = maya.now() + datetime.timedelta(days=5) +m = 1 n = 1 -uri = b"secret/files/and/stuff" +label = b"secret/files/and/stuff" -# Alice gets on the network and discovers Ursula, presumably from the blockchain. -ALICE.learn_about_nodes(address="https://localhost", port="3550") + +# Alice gets on the network and, knowing about at least one Ursula, +# Is able to discover all Ursulas. +ALICE.network_bootstrap([("localhost", 3601)]) # Alice grants to Bob. -policy = ALICE.grant(BOB, uri, networky_stuff, m=1, n=n, +policy = ALICE.grant(BOB, label, network_middleware, m=m, n=n, expiration=policy_end_datetime) -policy.publish_treasure_map(networky_stuff, use_dht=False) hrac, treasure_map = policy.hrac(), policy.treasure_map -# Bob learns about Ursula, gets the TreasureMap, and follows it. -BOB.learn_about_nodes(address="https://localhost", port="3550") -networky_stuff = NetworkyStuff() -BOB.get_treasure_map(policy, networky_stuff) -BOB.follow_treasure_map(hrac) +# Bob can re-assemble the hrac himself with knowledge he already has. +hrac = keccak_digest(bytes(ALICE.stamp) + bytes(BOB.stamp) + label) +BOB.join_policy(ALICE, hrac, node_list=[("localhost", 3601)]) # Now, Alice and Bob are ready for some throughput. - finnegans_wake = open(sys.argv[1], 'rb') start_time = datetime.datetime.now() @@ -55,16 +56,10 @@ for counter, plaintext in enumerate(finnegans_wake): print("PREs per second: {}".format(counter / seconds)) print("********************************") - ciphertext, capsule = pre.encrypt(ALICE.public_key(EncryptingPower), plaintext) + data_source = DataSource(policy_pubkey_enc=policy.public_key()) + message_kit, _signature = data_source.encapsulate_single_message(plaintext) - message_kit = MessageKit(ciphertext=ciphertext, capsule=capsule, - alice_pubkey=ALICE.public_key(EncryptingPower)) + delivered_cleartext = BOB.retrieve(hrac=hrac, message_kit=message_kit, data_source=data_source) - work_orders = BOB.generate_work_orders(hrac, capsule) - print(plaintext) - cfrags = BOB.get_reencrypted_c_frags(networky_stuff, work_orders[bytes(URSULA.stamp)]) - - capsule.attach_cfrag(cfrags[0]) - delivered_cleartext = pre.decrypt(capsule, BOB._crypto_power._power_ups[EncryptingPower].keypair._privkey, ciphertext, ALICE.public_key(EncryptingPower)) assert plaintext == delivered_cleartext print("Retrieved: {}".format(delivered_cleartext)) diff --git a/examples/run_ursula_with_rest_and_dht_but_no_mining.py b/examples/run_ursula_with_rest_and_dht_but_no_mining.py index 3d3e6956b..c43b160e6 100644 --- a/examples/run_ursula_with_rest_and_dht_but_no_mining.py +++ b/examples/run_ursula_with_rest_and_dht_but_no_mining.py @@ -9,7 +9,7 @@ import os from cryptography.hazmat.primitives.asymmetric import ec -from hendrix.deploy.ssl import HendrixDeployTLS +from hendrix.deploy.tls import HendrixDeployTLS from hendrix.facilities.services import ExistingKeyTLSContextFactory from nkms.characters import Ursula from OpenSSL.crypto import X509 @@ -19,14 +19,14 @@ from nkms.crypto.api import generate_self_signed_certificate DB_NAME = "non-mining-proxy-node" -_URSULA = Ursula(dht_port=3501, dht_interface="localhost", db_name=DB_NAME) -_URSULA.listen() +_URSULA = Ursula(dht_port=3501, rest_port=3601, ip_address="localhost", db_name=DB_NAME) +_URSULA.dht_listen() CURVE = ec.SECP256R1 cert, private_key = generate_self_signed_certificate(_URSULA.stamp.fingerprint().decode(), CURVE) deployer = HendrixDeployTLS("start", - {"wsgi":_URSULA.rest_app, "https_port": 3550}, + {"wsgi":_URSULA.rest_app, "https_port": _URSULA.rest_port}, key=private_key, cert=X509.from_cryptography(cert), context_factory=ExistingKeyTLSContextFactory, diff --git a/examples/sandbox_resources.py b/examples/sandbox_resources.py index e69de29bb..669a76dff 100644 --- a/examples/sandbox_resources.py +++ b/examples/sandbox_resources.py @@ -0,0 +1,32 @@ +import requests +from nkms.characters import Ursula +from nkms.network.node import NetworkyStuff +from nkms.crypto.powers import SigningPower, EncryptingPower + + +class SandboxNetworkyStuff(NetworkyStuff): + + def __init__(self, ursulas): + self.ursulas = ursulas + + def find_ursula(self, contract=None): + ursula = Ursula.as_discovered_on_network(dht_port=None, + ip_address="localhost", + rest_port=3601, + powers_and_keys={ + SigningPower: self.ursulas[0].stamp.as_umbral_pubkey(), + EncryptingPower: self.ursulas[0].public_key(EncryptingPower) + } + ) + response = requests.post("https://localhost:3601/consider_arrangement", bytes(contract), verify=False) + if response.status_code == 200: + response.was_accepted = True + else: + raise RuntimeError("Something went terribly wrong. What'd you do?!") + return ursula, response + + def enact_policy(self, ursula, hrac, payload): + endpoint = 'https://{}:{}/kFrag/{}'.format(ursula.ip_address, ursula.rest_port, hrac.hex()) + response = requests.post(endpoint, payload, verify=False) + # TODO: Something useful here and it's probably ready to go down into NetworkyStuff. + return response.status_code == 200 From 50915fc1f2338371bf486141a7b8a8a0373b0921 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 02:44:10 -0700 Subject: [PATCH 13/15] Removing "random" Ursulas from tests; selecting by rest_port instead. --- nkms/characters.py | 6 +-- tests/characters/test_bob_handles_frags.py | 31 +++++------- tests/utilities.py | 59 +++++++++------------- 3 files changed, 40 insertions(+), 56 deletions(-) diff --git a/nkms/characters.py b/nkms/characters.py index 8c8483bde..078641344 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -420,7 +420,7 @@ class Bob(Character): raise self.NotEnoughUrsulas( "Unable to follow the TreasureMap; we just don't know enough nodes to ask about this. Maybe try using the DHT instead.") - new_nodes = self.learn_about_nodes(node_to_check.rest_address, + new_nodes = self.learn_about_nodes(node_to_check.ip_address, node_to_check.rest_port) for new_node_pubkey in new_nodes.keys(): if new_node_pubkey in treasure_map: @@ -540,8 +540,8 @@ class Bob(Character): return generated_work_orders - def get_reencrypted_c_frags(self, networky_stuff, work_order): - cfrags = networky_stuff.reencrypt(work_order) + def get_reencrypted_c_frags(self, work_order): + cfrags = self.network_middleware.reencrypt(work_order) if not len(work_order) == len(cfrags): raise ValueError("Ursula gave back the wrong number of cfrags. She's up to something.") for counter, capsule in enumerate(work_order.capsules): diff --git a/tests/characters/test_bob_handles_frags.py b/tests/characters/test_bob_handles_frags.py index f396013dd..a644fa4d0 100644 --- a/tests/characters/test_bob_handles_frags.py +++ b/tests/characters/test_bob_handles_frags.py @@ -49,7 +49,7 @@ def test_bob_can_follow_treasure_map(enacted_policy, ursulas, bob, alice): assert total_known == len(enacted_policy.treasure_map) -def test_bob_can_follow_treasure_map_even_if_he_only_knows_of_one_node(enacted_policy, ursulas, bob): +def test_bob_can_follow_treasure_map_even_if_he_only_knows_of_one_node(enacted_policy, bob): # Again, let's assume that he received the TreasureMap via a side channel. hrac, treasure_map = enacted_policy.hrac(), enacted_policy.treasure_map bob.treasure_maps[hrac] = treasure_map @@ -69,7 +69,7 @@ def test_bob_can_follow_treasure_map_even_if_he_only_knows_of_one_node(enacted_p assert total_known == len(treasure_map) -def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice, bob, ursulas, +def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, bob, ursulas, capsule_side_channel): """ Now that Bob has his list of Ursulas, he can issue a WorkOrder to one. Upon receiving the WorkOrder, Ursula @@ -102,14 +102,10 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice, # And the Ursula. assert len(bob._saved_work_orders.ursulas) == 1 - networky_stuff = MockNetworkyStuff(ursulas) ursula_dht_key, work_order = list(work_orders.items())[0] - # In the real world, we'll have a full Ursula node here. But in this case, we need to fake it. - work_order.ursula = ursulas[0] - # **** RE-ENCRYPTION HAPPENS HERE! **** - cfrags = bob.get_reencrypted_c_frags(networky_stuff, work_order) + cfrags = bob.get_reencrypted_c_frags(work_order) # We only gave one Capsule, so we only got one cFrag. assert len(cfrags) == 1 @@ -123,7 +119,13 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice, # OK, so cool - Bob has his cFrag! Let's make sure everything went properly. First, we'll show that it is in fact # the correct cFrag (ie, that Ursula performed reencryption properly). - ursula = work_order.ursula + for u in ursulas: + if u.rest_port == work_order.ursula.rest_port: + ursula = u + break + else: + raise RuntimeError("Somehow we don't know about this Ursula. Major malfunction.") + kfrag_bytes = ursula.datastore.get_policy_arrangement( work_order.kfrag_hrac.hex().encode()).k_frag the_kfrag = KFrag.from_bytes(kfrag_bytes) @@ -136,7 +138,7 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_policy, alice, assert work_orders_from_bob[0] == work_order -def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_policy, alice, bob, +def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_policy, bob, ursulas, capsule_side_channel): # In our last episode, Bob made a WorkOrder for the capsule... assert len(bob._saved_work_orders.by_capsule(capsule_side_channel[0].capsule)) == 1 @@ -167,11 +169,7 @@ def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_polic assert new_work_order != saved_work_order # We can get a new CFrag, just like last time. - networky_stuff = MockNetworkyStuff(ursulas) - # In the real world, we'll have a full Ursula node here. But in this case, we need to fake it. - new_work_order.ursula = ursulas[1] - - cfrags = bob.get_reencrypted_c_frags(networky_stuff, new_work_order) + cfrags = bob.get_reencrypted_c_frags(new_work_order) # Again: one Capsule, one cFrag. assert len(cfrags) == 1 @@ -202,10 +200,7 @@ def test_bob_gathers_and_combines(enacted_policy, bob, ursulas, capsule_side_cha num_ursulas=number_left_to_collect) _id_of_yet_another_ursula, new_work_order = list(new_work_orders.items())[0] - networky_stuff = MockNetworkyStuff(ursulas) - # In the real world, we'll have a full Ursula node here. But in this case, we need to fake it. - new_work_order.ursula = ursulas[2] - cfrags = bob.get_reencrypted_c_frags(networky_stuff, new_work_order) + cfrags = bob.get_reencrypted_c_frags(new_work_order) the_message_kit.capsule.attach_cfrag(cfrags[0]) # Now. diff --git a/tests/utilities.py b/tests/utilities.py index 7bdb91064..57c62a81f 100644 --- a/tests/utilities.py +++ b/tests/utilities.py @@ -68,64 +68,53 @@ class MockNetworkyStuff(NetworkyStuff): raise self.NotEnoughQualifiedUrsulas mock_client = TestClient(ursula.rest_app) response = mock_client.post("http://localhost/consider_arrangement", bytes(arrangement)) + assert response.status_code == 200 return ursula, MockArrangementResponse() def enact_policy(self, ursula, hrac, payload): - mock_client = TestClient(ursula.rest_app) + rest_app = self._get_rest_app_by_port(ursula.rest_port) + mock_client = TestClient(rest_app) response = mock_client.post('http://localhost/kFrag/{}'.format(hrac.hex()), payload) + assert response.status_code == 200 return True, ursula.stamp.as_umbral_pubkey() + def _get_rest_app_by_port(self, port): + for ursula in self._ursulas.values(): + if ursula.rest_port == port: + rest_app = ursula.rest_app + break + else: + raise RuntimeError( + "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) + return rest_app + def send_work_order_payload_to_ursula(self, work_order): - mock_client = TestClient(work_order.ursula.rest_app) + rest_app = self._get_rest_app_by_port(work_order.ursula.rest_port) + mock_client = TestClient(rest_app) payload = work_order.payload() hrac_as_hex = work_order.kfrag_hrac.hex() return mock_client.post('http://localhost/kFrag/{}/reencrypt'.format(hrac_as_hex), payload) def get_treasure_map_from_node(self, node, map_id): - for ursula in self._ursulas.values(): - if ursula.rest_port == node.rest_port: - rest_app = ursula.rest_app - break - else: - raise RuntimeError( - "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) - mock_client = TestClient(ursula.rest_app) + rest_app = self._get_rest_app_by_port(node.rest_port) + mock_client = TestClient(rest_app) return mock_client.get("http://localhost/treasure_map/{}".format(map_id.hex())) def ursula_from_rest_interface(self, address, port): - for ursula in self._ursulas.values(): - if ursula.rest_port == port: - rest_app = ursula.rest_app - break - else: - raise RuntimeError( - "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) - mock_client = TestClient(ursula.rest_app) + rest_app = self._get_rest_app_by_port(port) + mock_client = TestClient(rest_app) response = mock_client.get("http://localhost/public_keys") return response def get_nodes_via_rest(self, address, port): - for ursula in self._ursulas.values(): - if ursula.rest_port == port: - rest_app = ursula.rest_app - break - else: - raise RuntimeError("Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) - mock_client = TestClient(ursula.rest_app) + rest_app = self._get_rest_app_by_port(port) + mock_client = TestClient(rest_app) response = mock_client.get("http://localhost/list_nodes") return response def push_treasure_map_to_node(self, node, map_id, map_payload): - port = node.rest_port - for ursula in self._ursulas.values(): - if ursula.rest_port == port: - rest_app = ursula.rest_app - break - else: - raise RuntimeError( - "Can't find an Ursula with port {} - did you spin up the right test ursulas?".format(port)) - mock_client = TestClient(ursula.rest_app) + rest_app = self._get_rest_app_by_port(node.rest_port) + mock_client = TestClient(rest_app) response = mock_client.post("http://localhost/treasure_map/{}".format(map_id.hex()), data=map_payload, verify=False) return response - From 7890dcc2b2b4d6cc8e9ac1c07cc32de8e06d2464 Mon Sep 17 00:00:00 2001 From: jMyles Date: Tue, 17 Apr 2018 21:55:25 -0700 Subject: [PATCH 14/15] Several touchups to Finnegan's Wake logic and the high-level APIs that power it. --- .../alicebob-grant-and-line-by-line-PRE.py | 105 +++++++++++++++--- nkms/characters.py | 44 +++++--- nkms/crypto/powers.py | 13 ++- nkms/data_sources.py | 19 +++- nkms/policy/models.py | 9 +- .../test_alice_can_grant_and_revoke.py | 12 +- tests/characters/test_bob_handles_frags.py | 2 +- tests/network/test_network_actors.py | 6 +- 8 files changed, 151 insertions(+), 59 deletions(-) diff --git a/examples/alicebob-grant-and-line-by-line-PRE.py b/examples/alicebob-grant-and-line-by-line-PRE.py index 03b6b5b2b..26e9b1806 100644 --- a/examples/alicebob-grant-and-line-by-line-PRE.py +++ b/examples/alicebob-grant-and-line-by-line-PRE.py @@ -1,25 +1,28 @@ # This is an example of Alice setting a Policy on the NuCypher network. # In this example, Alice uses n=1, which is almost always a bad idea. Don't do it. -# WIP w/ hendrix@8227c4abcb37ee6d27528a13ec22d55ee106107f +# WIP w/ hendrix@3.0.0 import datetime import sys from examples.sandbox_resources import SandboxNetworkyStuff from nkms.characters import Alice, Bob, Ursula -from nkms.crypto.api import keccak_digest from nkms.data_sources import DataSource from nkms.network.node import NetworkyStuff import maya -# Some basic setup. - -ALICE = Alice() -BOB = Bob() +# This is already running in another process. URSULA = Ursula.from_rest_url(NetworkyStuff(), address="localhost", port=3601) network_middleware = SandboxNetworkyStuff([URSULA]) + +######### +# Alice # +######### + +ALICE = Alice(network_middleware=network_middleware) + # Here are our Policy details. policy_end_datetime = maya.now() + datetime.timedelta(days=5) m = 1 @@ -32,17 +35,43 @@ label = b"secret/files/and/stuff" ALICE.network_bootstrap([("localhost", 3601)]) # Alice grants to Bob. -policy = ALICE.grant(BOB, label, network_middleware, m=m, n=n, +BOB = Bob() +policy = ALICE.grant(BOB, label, m=m, n=n, expiration=policy_end_datetime) -hrac, treasure_map = policy.hrac(), policy.treasure_map -# Bob can re-assemble the hrac himself with knowledge he already has. -hrac = keccak_digest(bytes(ALICE.stamp) + bytes(BOB.stamp) + label) -BOB.join_policy(ALICE, hrac, node_list=[("localhost", 3601)]) +# Alice puts her public key somewhere for Bob to find later... +alices_pubkey_saved_for_posterity = bytes(ALICE.stamp) -# Now, Alice and Bob are ready for some throughput. +# ...and then disappears from the internet. +del ALICE +# (this is optional of course - she may wish to remain in order to create +# new policies in the future. The point is - she is no longer obligated. + +##################### +# some time passes. # +# ... # +# And now for Bob. # +##################### + +# Bob wants to join the policy so that he can receive any future +# data shared on it. +# He needs a few piece of knowledge to do that. +BOB.join_policy(label, # The label - he needs to know what data he's after. + alices_pubkey_saved_for_posterity, # To verify the signature, he'll need Alice's public key. + verify_sig=True, # And yes, he usually wants to verify that signature. + # He can also bootstrap himself onto the network more quickly + # by providing a list of known nodes at this time. + node_list=[("localhost", 3601)] + ) + +# Now that Bob has joined the Policy, let's show how DataSources +# can share data with the members of this Policy and then how Bob retrieves it. finnegans_wake = open(sys.argv[1], 'rb') +# We'll also keep track of some metadata to gauge performance. +# You can safely ignore from here until... +################################################################################ + start_time = datetime.datetime.now() for counter, plaintext in enumerate(finnegans_wake): @@ -56,10 +85,58 @@ for counter, plaintext in enumerate(finnegans_wake): print("PREs per second: {}".format(counter / seconds)) print("********************************") - data_source = DataSource(policy_pubkey_enc=policy.public_key()) + +################################################################################ +# ...here. OK, pay attention again. +# Now it's time for... + + ##################### + # Using DataSources # + ##################### + + # Now Alice has set a Policy and Bob has joined it. + # You're ready to make some DataSources and encrypt for Bob. + + # It may also be helpful to imagine that you have multiple Bobs, + # multiple Labels, or both. + + # First we make a DataSource for this policy. + data_source = DataSource(policy_pubkey_enc=policy.public_key) + + # Here's how we generate a MessageKit for the Policy. We also get a signature + # here, which can be passed via a side-channel (or posted somewhere public as + # testimony) and verified if desired. In this case, the plaintext is a + # single passage from James Joyce's Finnegan's Wake. + # The matter of whether encryption makes the passage more or less readable + # is left to the reader to determine. message_kit, _signature = data_source.encapsulate_single_message(plaintext) - delivered_cleartext = BOB.retrieve(hrac=hrac, message_kit=message_kit, data_source=data_source) + # The DataSource will want to be able to be verified by Bob, so it leaves + # its Public Key somewhere. + data_source_public_key = bytes(data_source.stamp) + # It can save the MessageKit somewhere (IPFS, etc) and then it too can + # choose to disappear (although it may also opt to continue transmitting + # as many messages as may be appropriate). + del data_source + + ############### + # Back to Bob # + ############### + + # Bob needs to reconstruct the DataSource. + datasource_as_understood_by_bob = DataSource.from_public_keys( + policy_public_key=policy.public_key, + datasource_public_key=data_source_public_key, + label=label + ) + + # Now Bob can retrieve the original message. He just needs the MessageKit + # and the DataSource which produced it. + delivered_cleartext = BOB.retrieve(message_kit=message_kit, + data_source=datasource_as_understood_by_bob, + alice_pubkey_sig=alices_pubkey_saved_for_posterity) + + # We show that indeed this is the passage originally encrypted by the DataSource. assert plaintext == delivered_cleartext print("Retrieved: {}".format(delivered_cleartext)) diff --git a/nkms/characters.py b/nkms/characters.py index 078641344..54684fb04 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -304,7 +304,8 @@ class Character(object): powers_and_keys=({SigningPower: pubkey}) ) else: - self.log.warn("Discovered node with bad signature: {}".format(node_meta)) + message = "Suspicious Activity: Discovered node with bad signature: {}. Propagated by: {}:{}".format(node_meta, address, port) + self.log.warn(message) return new_nodes def network_bootstrap(self, node_list): @@ -355,8 +356,7 @@ class Alice(Character, PolicyAuthor): return policy - def grant(self, bob, uri, networky_stuff, - m=None, n=None, expiration=None, deposit=None): + def grant(self, bob, uri, m=None, n=None, expiration=None, deposit=None): if not m: # TODO: get m from config #176 raise NotImplementedError @@ -369,7 +369,7 @@ class Alice(Character, PolicyAuthor): if not deposit: default_deposit = None # TODO: Check default deposit in config. #176 if not default_deposit: - deposit = networky_stuff.get_competitive_rate() + deposit = self.network_middleware.get_competitive_rate() if deposit == NotImplemented: deposit = constants.NON_PAYMENT(b"0000000") @@ -379,12 +379,12 @@ class Alice(Character, PolicyAuthor): # by trying differet # deposits and expirations on a limited number of Ursulas. # Users may decide to inject some market strategies here. - found_ursulas = policy.find_ursulas(networky_stuff, deposit, + found_ursulas = policy.find_ursulas(self.network_middleware, deposit, expiration, num_ursulas=n) policy.match_kfrags_to_found_ursulas(found_ursulas) # REST call happens here, as does population of TreasureMap. - policy.enact(networky_stuff) - policy.publish_treasure_map(networky_stuff) + policy.enact(self.network_middleware) + policy.publish_treasure_map(self.network_middleware) return policy # Now with TreasureMap affixed! @@ -456,8 +456,8 @@ class Bob(Character): powers_and_keys=({SigningPower: ursula_pubkey_sig}) ) - def get_treasure_map(self, alice, hrac, using_dht=False): - map_id = keccak_digest(bytes(alice.stamp) + hrac) + def get_treasure_map(self, alice_pubkey_sig, hrac, using_dht=False, verify_sig=True): + map_id = keccak_digest(alice_pubkey_sig + hrac) if using_dht: ursula_coro = self.server.get(map_id) @@ -470,10 +470,14 @@ class Bob(Character): tmap_message_kit = self.get_treasure_map_from_known_ursulas(self.network_middleware, map_id) - verified, packed_node_list = self.verify_from( - alice, tmap_message_kit, - decrypt=True - ) + if verify_sig: + alice = Alice.from_public_keys({SigningPower: alice_pubkey_sig}) + verified, packed_node_list = self.verify_from( + alice, tmap_message_kit, + decrypt=True + ) + else: + assert False if not verified: return constants.NOT_FROM_ALICE @@ -554,14 +558,18 @@ class Bob(Character): def get_ursula(self, ursula_id): return self._ursulas[ursula_id] - def join_policy(self, alice, hrac, using_dht=False, node_list=None): - # TODO: unfuckify this + def join_policy(self, policy_pubkey, label, alice_pubkey_sig=None, + using_dht=False, node_list=None, verify_sig=True): + if verify_sig and not alice_pubkey_sig: + raise ValueError("Can't verify the signature without Alice's key.") + hrac = keccak_digest(bytes(policy_pubkey) + bytes(self.stamp) + label) if node_list: self.network_bootstrap(node_list) - self.get_treasure_map(alice, hrac, using_dht=using_dht) + self.get_treasure_map(alice_pubkey_sig, hrac, using_dht=using_dht, verify_sig=verify_sig) self.follow_treasure_map(hrac, using_dht=using_dht) - def retrieve(self, hrac, message_kit, data_source): + def retrieve(self, message_kit, data_source): + hrac = keccak_digest(bytes(data_source.policy_pubkey) + self.stamp + data_source.label) treasure_map = self.treasure_maps[hrac] # First, a quick sanity check to make sure we know about at least m nodes. @@ -574,7 +582,7 @@ class Bob(Character): work_orders = self.generate_work_orders(hrac, message_kit.capsule) for node_id in self.treasure_maps[hrac]: node = self.known_nodes[UmbralPublicKey.from_bytes(node_id)] - cfrags = self.get_reencrypted_c_frags(self.network_middleware, work_orders[bytes(node.stamp)]) + cfrags = self.get_reencrypted_c_frags(work_orders[bytes(node.stamp)]) message_kit.capsule.attach_cfrag(cfrags[0]) verified, delivered_cleartext = self.verify_from(data_source, message_kit, decrypt=True) diff --git a/nkms/crypto/powers.py b/nkms/crypto/powers.py index 0c04f51ac..41c1444b6 100644 --- a/nkms/crypto/powers.py +++ b/nkms/crypto/powers.py @@ -1,5 +1,4 @@ import inspect - from nkms.keystore import keypairs from nkms.keystore.keypairs import SigningKeypair, EncryptingKeypair from umbral.keys import UmbralPublicKey, UmbralPrivateKey @@ -75,10 +74,16 @@ class KeyPairBasedPower(CryptoPowerUp): elif keypair: self.keypair = keypair else: - # They didn't pass a keypair; we'll make one with the bytes (if any) - # they provided. + # They didn't pass a keypair; we'll make one with the bytes or + # UmbralPublicKey if they provided such a thing. if pubkey: - key_to_pass_to_keypair = pubkey + try: + key_to_pass_to_keypair = pubkey.as_umbral_pubkey() + except AttributeError: + try: + key_to_pass_to_keypair = UmbralPublicKey.from_bytes(pubkey) + except TypeError: + key_to_pass_to_keypair = pubkey else: # They didn't even pass pubkey_bytes. We'll generate a keypair. key_to_pass_to_keypair = UmbralPrivateKey.gen_key() diff --git a/nkms/data_sources.py b/nkms/data_sources.py index d57c3c855..03e066ed0 100644 --- a/nkms/data_sources.py +++ b/nkms/data_sources.py @@ -2,18 +2,29 @@ from nkms.crypto.api import encrypt_and_sign from nkms.crypto.signature import SignatureStamp from nkms.keystore.keypairs import SigningKeypair from constant_sorrow.constants import NO_SIGNING_POWER +from umbral.keys import UmbralPublicKey + class DataSource: - def __init__(self, policy_pubkey_enc, signer=NO_SIGNING_POWER): - self._policy_pubkey_enc = policy_pubkey_enc + def __init__(self, policy_pubkey_enc, signer=NO_SIGNING_POWER, label=None): + self.policy_pubkey = policy_pubkey_enc if signer is NO_SIGNING_POWER: signer = SignatureStamp(SigningKeypair()) # TODO: Generate signing key properly. #241 self.stamp = signer + self.label = label def encapsulate_single_message(self, message): - message_kit, signature = encrypt_and_sign(self._policy_pubkey_enc, + message_kit, signature = encrypt_and_sign(self.policy_pubkey, plaintext=message, signer=self.stamp) - message_kit.policy_pubkey = self._policy_pubkey_enc # TODO: We can probably do better here. + message_kit.policy_pubkey = self.policy_pubkey # TODO: We can probably do better here. return message_kit, signature + + @classmethod + def from_public_keys(cls, policy_public_key, datasource_public_key, label): + umbral_public_key = UmbralPublicKey.from_bytes(datasource_public_key) + return cls(policy_public_key, + signer=SignatureStamp(SigningKeypair(umbral_public_key)), + label=label, + ) diff --git a/nkms/policy/models.py b/nkms/policy/models.py index 2649f4ff7..b0aa13687 100644 --- a/nkms/policy/models.py +++ b/nkms/policy/models.py @@ -165,13 +165,6 @@ class Policy(object): return policy def hrac(self): - """ - A convenience method for generating an hrac for this instance. - """ - return self.hrac_for(self.alice, self.bob, self.uri) - - @staticmethod - def hrac_for(alice, bob, uri): """ The "hashed resource authentication code". @@ -183,7 +176,7 @@ class Policy(object): Alice and Bob have all the information they need to construct this. Ursula does not, so we share it with her. """ - return keccak_digest(bytes(alice.stamp) + bytes(bob.stamp) + uri) + return keccak_digest(bytes(self.public_key()) + bytes(self.bob.stamp) + self.uri) def treasure_map_dht_key(self): """ diff --git a/tests/characters/test_alice_can_grant_and_revoke.py b/tests/characters/test_alice_can_grant_and_revoke.py index 1e0e24bb6..0a0c3d62d 100644 --- a/tests/characters/test_alice_can_grant_and_revoke.py +++ b/tests/characters/test_alice_can_grant_and_revoke.py @@ -7,19 +7,17 @@ from nkms.crypto.api import keccak_digest from nkms.crypto.constants import PUBLIC_KEY_LENGTH from nkms.crypto.powers import SigningPower, EncryptingPower from bytestring_splitter import BytestringSplitter -from tests.utilities import MockNetworkyStuff from umbral.fragments import KFrag from umbral.keys import UmbralPublicKey import maya -def test_grant(alice, bob, ursulas): - networky_stuff = MockNetworkyStuff(ursulas) +def test_grant(alice, bob): policy_end_datetime = maya.now() + datetime.timedelta(days=5) n = 5 uri = b"this_is_the_path_to_which_access_is_being_granted" - policy = alice.grant(bob, uri, networky_stuff, m=3, n=n, - expiration=policy_end_datetime) + policy = alice.grant(bob, uri, m=3, n=n, + expiration=policy_end_datetime) # The number of policies is equal to the number of Ursulas we're using (n) assert len(policy._accepted_arrangements) == n @@ -28,7 +26,7 @@ def test_grant(alice, bob, ursulas): ursula = list(policy._accepted_arrangements.values())[0].ursula # Get the Policy from Ursula's datastore, looking up by hrac. - proper_hrac = keccak_digest(bytes(alice.stamp) + bytes(bob.stamp) + uri) + proper_hrac = keccak_digest(bytes(policy.public_key()) + bytes(bob.stamp) + uri) retrieved_policy = ursula.datastore.get_policy_arrangement(proper_hrac.hex().encode()) # TODO: Make this a legit KFrag, not bytes. @@ -42,7 +40,7 @@ def test_grant(alice, bob, ursulas): assert found -def test_alice_can_get_ursulas_keys_via_rest(alice, ursulas): +def test_alice_can_get_ursulas_keys_via_rest(ursulas): mock_client = TestClient(ursulas[0].rest_app) response = mock_client.get('http://localhost/public_keys') splitter = BytestringSplitter( diff --git a/tests/characters/test_bob_handles_frags.py b/tests/characters/test_bob_handles_frags.py index a644fa4d0..6c6cf2860 100644 --- a/tests/characters/test_bob_handles_frags.py +++ b/tests/characters/test_bob_handles_frags.py @@ -39,7 +39,7 @@ def test_bob_can_follow_treasure_map(enacted_policy, ursulas, bob, alice): assert len(bob.known_nodes) == len(ursulas) # Now, Bob can get the TreasureMap all by himself, and doesn't need a side channel. - bob.get_treasure_map(alice, hrac) + bob.get_treasure_map(alice.stamp, hrac) newly_discovered, total_known = bob.follow_treasure_map(hrac) # He finds that he didn't need to discover any new nodes... diff --git a/tests/network/test_network_actors.py b/tests/network/test_network_actors.py index bd5f688ac..f5362e3ce 100644 --- a/tests/network/test_network_actors.py +++ b/tests/network/test_network_actors.py @@ -84,7 +84,7 @@ def test_alice_creates_policy_group_with_correct_hrac(idle_policy): bob = idle_policy.bob assert idle_policy.hrac() == keccak_digest( - bytes(alice.stamp) + bytes(bob.stamp) + alice.__resource_id) + bytes(idle_policy.public_key()) + bytes(bob.stamp) + alice.__resource_id) def test_alice_sets_treasure_map_on_network(enacted_policy, ursulas): @@ -154,13 +154,13 @@ def test_bob_can_retreive_the_treasure_map_and_decrypt_it(enacted_policy, ursula # If Bob doesn't know about any Ursulas, he can't find the TreasureMap via the REST swarm: with pytest.raises(bob.NotEnoughUrsulas): - treasure_map_from_wire = bob.get_treasure_map(enacted_policy.alice, enacted_policy.hrac()) + treasure_map_from_wire = bob.get_treasure_map(enacted_policy.alice.stamp, enacted_policy.hrac()) # Let's imagine he has learned about some - say, from the blockchain. bob.known_nodes = {u.interface_info_with_metadata(): u for u in ursulas} # Now try. - treasure_map_from_wire = bob.get_treasure_map(enacted_policy.alice, enacted_policy.hrac()) + treasure_map_from_wire = bob.get_treasure_map(enacted_policy.alice.stamp, enacted_policy.hrac()) assert enacted_policy.treasure_map == treasure_map_from_wire From f4d43470d2987cc290458e8d346895b90c4afcde Mon Sep 17 00:00:00 2001 From: jMyles Date: Wed, 18 Apr 2018 23:24:19 -0700 Subject: [PATCH 15/15] DelegatingPower now derives its Keypairs from a label. --- nkms/characters.py | 27 ++++++------ nkms/crypto/powers.py | 41 +++++++++++++++---- nkms/keystore/keypairs.py | 15 ------- nkms/policy/models.py | 17 ++++---- .../test_alice_can_grant_and_revoke.py | 2 +- tests/fixtures.py | 2 +- tests/network/test_network_actors.py | 2 +- 7 files changed, 55 insertions(+), 51 deletions(-) diff --git a/nkms/characters.py b/nkms/characters.py index 54684fb04..968d5b44d 100644 --- a/nkms/characters.py +++ b/nkms/characters.py @@ -73,11 +73,9 @@ class Character(object): if crypto_power: self._crypto_power = crypto_power elif crypto_power_ups: - self._crypto_power = CryptoPower(power_ups=crypto_power_ups, - generate_keys_if_needed=is_me) + self._crypto_power = CryptoPower(power_ups=crypto_power_ups) else: - self._crypto_power = CryptoPower(self._default_crypto_powerups, - generate_keys_if_needed=is_me) + self._crypto_power = CryptoPower(self._default_crypto_powerups) if is_me: self.network_middleware = network_middleware or NetworkyStuff() try: @@ -326,7 +324,7 @@ class Alice(Character, PolicyAuthor): super().__init__(*args, **kwargs) PolicyAuthor.__init__(self, self.address, policy_agent=FakePolicyAgent()) - def generate_kfrags(self, bob, m, n) -> List: + def generate_kfrags(self, bob, label, m, n) -> List: """ Generates re-encryption key frags ("KFrags") and returns them. @@ -337,20 +335,21 @@ class Alice(Character, PolicyAuthor): :param n: Total number of kfrags to generate """ bob_pubkey_enc = bob.public_key(EncryptingPower) - return self._crypto_power.power_ups(DelegatingPower).generate_kfrags(bob_pubkey_enc, m, n) + return self._crypto_power.power_ups(DelegatingPower).generate_kfrags(bob_pubkey_enc, label, m, n) - def create_policy(self, bob: "Bob", uri: bytes, m: int, n: int): + def create_policy(self, bob: "Bob", label: bytes, m: int, n: int): """ Create a Policy to share uri with bob. Generates KFrags and attaches them. """ - kfrags = self.generate_kfrags(bob, m, n) + public_key, kfrags = self.generate_kfrags(bob, label, m, n) from nkms.policy.models import Policy policy = Policy.from_alice( alice=self, + label=label, bob=bob, kfrags=kfrags, - uri=uri, + public_key=public_key, m=m, ) @@ -558,18 +557,16 @@ class Bob(Character): def get_ursula(self, ursula_id): return self._ursulas[ursula_id] - def join_policy(self, policy_pubkey, label, alice_pubkey_sig=None, + def join_policy(self, label, alice_pubkey_sig, using_dht=False, node_list=None, verify_sig=True): - if verify_sig and not alice_pubkey_sig: - raise ValueError("Can't verify the signature without Alice's key.") - hrac = keccak_digest(bytes(policy_pubkey) + bytes(self.stamp) + label) + hrac = keccak_digest(bytes(alice_pubkey_sig) + bytes(self.stamp) + label) if node_list: self.network_bootstrap(node_list) self.get_treasure_map(alice_pubkey_sig, hrac, using_dht=using_dht, verify_sig=verify_sig) self.follow_treasure_map(hrac, using_dht=using_dht) - def retrieve(self, message_kit, data_source): - hrac = keccak_digest(bytes(data_source.policy_pubkey) + self.stamp + data_source.label) + def retrieve(self, message_kit, data_source, alice_pubkey_sig): + hrac = keccak_digest(bytes(alice_pubkey_sig) + self.stamp + data_source.label) treasure_map = self.treasure_maps[hrac] # First, a quick sanity check to make sure we know about at least m nodes. diff --git a/nkms/crypto/powers.py b/nkms/crypto/powers.py index 41c1444b6..fc0d251fe 100644 --- a/nkms/crypto/powers.py +++ b/nkms/crypto/powers.py @@ -1,7 +1,10 @@ import inspect +from typing import List, Union + from nkms.keystore import keypairs from nkms.keystore.keypairs import SigningKeypair, EncryptingKeypair -from umbral.keys import UmbralPublicKey, UmbralPrivateKey +from umbral.keys import UmbralPublicKey, UmbralPrivateKey, UmbralKeyingMaterial +from umbral import pre class PowerUpError(TypeError): @@ -17,11 +20,10 @@ class NoEncryptingPower(PowerUpError): class CryptoPower(object): - def __init__(self, power_ups=None, generate_keys_if_needed=False): + def __init__(self, power_ups=None): self._power_ups = {} # TODO: The keys here will actually be IDs for looking up in a KeyStore. self.public_keys = {} - self.generate_keys = generate_keys_if_needed if power_ups is not None: for power_up in power_ups: @@ -35,8 +37,7 @@ class CryptoPower(object): power_up_instance = power_up elif CryptoPowerUp in inspect.getmro(power_up): power_up_class = power_up - power_up_instance = power_up( - generate_keys_if_needed=self.generate_keys) + power_up_instance = power_up() else: raise TypeError( ("power_up must be a subclass of CryptoPowerUp or an instance " @@ -106,6 +107,13 @@ class KeyPairBasedPower(CryptoPowerUp): return self.keypair.pubkey +class DerivedKeyBasedPower(CryptoPowerUp): + """ + Rather than rely on an established KeyPair, this type of power + derives a key at moments defined by the user. + """ + + class SigningPower(KeyPairBasedPower): _keypair_class = SigningKeypair not_found_error = NoSigningPower @@ -118,8 +126,23 @@ class EncryptingPower(KeyPairBasedPower): provides = ("decrypt",) -class DelegatingPower(KeyPairBasedPower): - _keypair_class = EncryptingKeypair - not_found_error = PowerUpError - provides = ("generate_kfrags",) +class DelegatingPower(DerivedKeyBasedPower): + def __init__(self): + self.umbral_keying_material = UmbralKeyingMaterial() + + def generate_kfrags(self, bob_pubkey_enc, label, m, n) -> Union[UmbralPublicKey, List]: + """ + Generates re-encryption key frags ("KFrags") and returns them. + + These KFrags can be used by Ursula to re-encrypt a Capsule for Bob so + that he can activate the Capsule. + :param bob_pubkey_enc: Bob's public key + :param m: Minimum number of KFrags needed to rebuild ciphertext + :param n: Total number of rekey shares to generate + """ + # TODO: salt? + + __private_key = self.umbral_keying_material.derive_privkey_by_label(label) + kfrags = pre.split_rekey(__private_key, bob_pubkey_enc, m, n) + return __private_key.get_pubkey(), kfrags diff --git a/nkms/keystore/keypairs.py b/nkms/keystore/keypairs.py index aeff106de..46b0a7c8a 100644 --- a/nkms/keystore/keypairs.py +++ b/nkms/keystore/keypairs.py @@ -8,7 +8,6 @@ from umbral import pre from umbral.config import default_curve from nkms.crypto.kits import MessageKit from nkms.crypto.signature import Signature -from typing import List class Keypair(object): @@ -80,20 +79,6 @@ class EncryptingKeypair(Keypair): return cleartext - def generate_kfrags(self, bob_pubkey_enc, m, n) -> List: - """ - Generates re-encryption key frags ("KFrags") and returns them. - - These KFrags can be used by Ursula to re-encrypt a Capsule for Bob so - that he can activate the Capsule. - :param bob_pubkey_enc: Bob's public key - :param m: Minimum number of KFrags needed to rebuild ciphertext - :param n: Total number of rekey shares to generate - """ - alice_priv_enc = self._privkey - kfrags = pre.split_rekey(alice_priv_enc, bob_pubkey_enc, m, n) - return kfrags - class SigningKeypair(Keypair): """ diff --git a/nkms/policy/models.py b/nkms/policy/models.py index b0aa13687..8f992cca0 100644 --- a/nkms/policy/models.py +++ b/nkms/policy/models.py @@ -114,17 +114,18 @@ class Policy(object): """ _ursula = None - def __init__(self, alice, bob=None, kfrags=(constants.UNKNOWN_KFRAG,), - label=None, m=None, alices_signature=constants.NOT_SIGNED): + def __init__(self, alice, label, bob=None, kfrags=(constants.UNKNOWN_KFRAG,), + public_key=None, m=None, alices_signature=constants.NOT_SIGNED): """ :param kfrags: A list of KFrags to distribute per this Policy. :param label: The identity of the resource to which Bob is granted access. """ self.alice = alice + self.label = label self.bob = bob self.kfrags = kfrags - self.uri = label + self.public_key = public_key self.treasure_map = TreasureMap(m=m) self._accepted_arrangements = OrderedDict() @@ -155,12 +156,13 @@ class Policy(object): @staticmethod def from_alice(kfrags, alice, + label, bob, - uri, + public_key, m, ): # TODO: What happened to Alice's signature - don't we include it here? - policy = Policy(alice, bob, kfrags, uri, m) + policy = Policy(alice, label, bob, kfrags, public_key, m) return policy @@ -176,7 +178,7 @@ class Policy(object): Alice and Bob have all the information they need to construct this. Ursula does not, so we share it with her. """ - return keccak_digest(bytes(self.public_key()) + bytes(self.bob.stamp) + self.uri) + return keccak_digest(bytes(self.alice.stamp) + bytes(self.bob.stamp) + self.label) def treasure_map_dht_key(self): """ @@ -267,9 +269,6 @@ class Policy(object): arrangement.publish(kfrag, ursula, result) # TODO: What if there weren't enough Arrangements approved to distribute n kfrags? We need to raise NotEnoughQualifiedUrsulas. - def public_key(self): - return self.alice.public_key(DelegatingPower) - class TreasureMap(object): def __init__(self, m, ursula_interface_ids=None): diff --git a/tests/characters/test_alice_can_grant_and_revoke.py b/tests/characters/test_alice_can_grant_and_revoke.py index 0a0c3d62d..fb9c2fdcc 100644 --- a/tests/characters/test_alice_can_grant_and_revoke.py +++ b/tests/characters/test_alice_can_grant_and_revoke.py @@ -26,7 +26,7 @@ def test_grant(alice, bob): ursula = list(policy._accepted_arrangements.values())[0].ursula # Get the Policy from Ursula's datastore, looking up by hrac. - proper_hrac = keccak_digest(bytes(policy.public_key()) + bytes(bob.stamp) + uri) + proper_hrac = keccak_digest(bytes(alice.stamp) + bytes(bob.stamp) + uri) retrieved_policy = ursula.datastore.get_policy_arrangement(proper_hrac.hex().encode()) # TODO: Make this a legit KFrag, not bytes. diff --git a/tests/fixtures.py b/tests/fixtures.py index 07ce1a866..7a5489979 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -93,7 +93,7 @@ def test_keystore(): @pytest.fixture(scope="module") def capsule_side_channel(enacted_policy): signing_keypair = SigningKeypair() - data_source = DataSource(policy_pubkey_enc=enacted_policy.public_key(), + data_source = DataSource(policy_pubkey_enc=enacted_policy.public_key, signer=SignatureStamp(signing_keypair)) message_kit, _signature = data_source.encapsulate_single_message(b"Welcome to the flippering.") return message_kit, data_source diff --git a/tests/network/test_network_actors.py b/tests/network/test_network_actors.py index f5362e3ce..867b34bf8 100644 --- a/tests/network/test_network_actors.py +++ b/tests/network/test_network_actors.py @@ -84,7 +84,7 @@ def test_alice_creates_policy_group_with_correct_hrac(idle_policy): bob = idle_policy.bob assert idle_policy.hrac() == keccak_digest( - bytes(idle_policy.public_key()) + bytes(bob.stamp) + alice.__resource_id) + bytes(alice.stamp) + bytes(bob.stamp) + alice.__resource_id) def test_alice_sets_treasure_map_on_network(enacted_policy, ursulas):