Merge pull request #446 from jMyles/character-crypto

Cleaned up cert handling; demo runs again.
pull/455/head
Tux 2018-09-24 13:52:22 -06:00 committed by GitHub
commit 27fc4d4d50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 339 additions and 51 deletions

View File

@ -0,0 +1,177 @@
# This is an example of Alice setting a Policy on the NuCypher network.
# In this example, Alice uses n=3.
# WIP w/ hendrix@3.1.0
import binascii
import datetime
import logging
import shutil
import sys
import os
import maya
from nucypher.characters.lawful import Alice, Bob, Ursula
from nucypher.data_sources import DataSource
# This is already running in another process.
from nucypher.network.middleware import RestMiddleware
from umbral.keys import UmbralPublicKey
##
# Boring setup stuff.
##
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
teacher_rest_port = sys.argv[2]
with open("examples-runtime-cruft/node-metadata-{}".format(teacher_rest_port), "r") as f:
f.seek(0)
teacher_bytes = binascii.unhexlify(f.read())
URSULA = Ursula.from_bytes(teacher_bytes, federated_only=True)
print("Will learn from {}".format(URSULA))
SHARED_CRUFTSPACE = "{}/examples-runtime-cruft".format(os.path.dirname(os.path.abspath(__file__)))
CRUFTSPACE = "{}/finnegans-wake-demo".format(SHARED_CRUFTSPACE)
CERTIFICATE_DIR = "{}/certs".format(CRUFTSPACE)
shutil.rmtree(CRUFTSPACE, ignore_errors=True)
os.mkdir(CRUFTSPACE)
os.mkdir(CERTIFICATE_DIR)
URSULA.save_certificate_to_disk(CERTIFICATE_DIR)
#########
# Alice #
#########
ALICE = Alice(network_middleware=RestMiddleware(),
known_nodes=(URSULA,),
federated_only=True,
always_be_learning=True,
known_certificates_dir=CERTIFICATE_DIR,
) # TODO: 289
# Here are our Policy details.
policy_end_datetime = maya.now() + datetime.timedelta(days=5)
m = 2
n = 3
label = b"secret/files/and/stuff"
# Alice grants to Bob.
BOB = Bob(known_nodes=(URSULA,),
federated_only=True,
always_be_learning=True,
known_certificates_dir=CERTIFICATE_DIR)
ALICE.start_learning_loop(now=True)
policy = ALICE.grant(BOB, label, m=m, n=n,
expiration=policy_end_datetime)
# Alice puts her public key somewhere for Bob to find later...
alices_pubkey_bytes_saved_for_posterity = bytes(ALICE.stamp)
# ...and then disappears from the internet.
del ALICE
# (this is optional of course - she may wish to remain in order to create
# new policies in the future. The point is - she is no longer obligated.
#####################
# some time passes. #
# ... #
# #
# ... #
# And now for Bob. #
#####################
# Bob wants to join the policy so that he can receive any future
# data shared on it.
# He needs a few pieces of knowledge to do that.
BOB.join_policy(label, # The label - he needs to know what data he's after.
alices_pubkey_bytes_saved_for_posterity, # To verify the signature, he'll need Alice's public key.
# He can also bootstrap himself onto the network more quickly
# by providing a list of known nodes at this time.
node_list=[("localhost", 3601)]
)
# Now that Bob has joined the Policy, let's show how DataSources
# can share data with the members of this Policy and then how Bob retrieves it.
finnegans_wake = open(sys.argv[1], 'rb')
# We'll also keep track of some metadata to gauge performance.
# You can safely ignore from here until...
################################################################################
start_time = datetime.datetime.now()
for counter, plaintext in enumerate(finnegans_wake):
if counter % 20 == 0:
now_time = datetime.datetime.now()
time_delta = now_time - start_time
seconds = time_delta.total_seconds()
print("********************************")
print("Performed {} PREs".format(counter))
print("Elapsed: {}".format(time_delta.total_seconds()))
print("PREs per second: {}".format(counter / seconds))
print("********************************")
################################################################################
# ...here. OK, pay attention again.
# Now it's time for...
#####################
# Using DataSources #
#####################
# Now Alice has set a Policy and Bob has joined it.
# You're ready to make some DataSources and encrypt for Bob.
# It may also be helpful to imagine that you have multiple Bobs,
# multiple Labels, or both.
# First we make a DataSource for this policy.
data_source = DataSource(policy_pubkey_enc=policy.public_key)
# Here's how we generate a MessageKit for the Policy. We also get a signature
# here, which can be passed via a side-channel (or posted somewhere public as
# testimony) and verified if desired. In this case, the plaintext is a
# single passage from James Joyce's Finnegan's Wake.
# The matter of whether encryption makes the passage more or less readable
# is left to the reader to determine.
message_kit, _signature = data_source.encapsulate_single_message(plaintext)
# The DataSource will want to be able to be verified by Bob, so it leaves
# its Public Key somewhere.
data_source_public_key = bytes(data_source.stamp)
# It can save the MessageKit somewhere (IPFS, etc) and then it too can
# choose to disappear (although it may also opt to continue transmitting
# as many messages as may be appropriate).
del data_source
###############
# Back to Bob #
###############
# Bob needs to reconstruct the DataSource.
datasource_as_understood_by_bob = DataSource.from_public_keys(
policy_public_key=policy.public_key,
datasource_public_key=data_source_public_key,
label=label
)
# Now Bob can retrieve the original message. He just needs the MessageKit
# and the DataSource which produced it.
alice_pubkey_restored_from_ancient_scroll = UmbralPublicKey.from_bytes(alices_pubkey_bytes_saved_for_posterity)
delivered_cleartexts = BOB.retrieve(message_kit=message_kit,
data_source=datasource_as_understood_by_bob,
alice_verifying_key=alice_pubkey_restored_from_ancient_scroll)
# We show that indeed this is the passage originally encrypted by the DataSource.
assert plaintext == delivered_cleartexts[0]
print("Retrieved: {}".format(delivered_cleartexts[0]))

View File

@ -0,0 +1,81 @@
# This is not an actual mining script. Don't use this to mine - you won't
# perform any re-encryptions, and you won't get paid.
# It might be (but might not be) useful for determining whether you have
# the proper depedencies and configuration to run an actual mining node.
# WIP w/ hendrix@tags/3.3.0rc1
import asyncio
import binascii
import logging
import os
import shutil
import sys
from nucypher.characters.lawful import Ursula
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
MY_REST_PORT = sys.argv[1]
# TODO: Use real path tooling here.
SHARED_CRUFTSPACE = "{}/examples-runtime-cruft".format(os.path.dirname(os.path.abspath(__file__)))
CRUFTSPACE = "{}/{}".format(SHARED_CRUFTSPACE, MY_REST_PORT)
DB_NAME = "{}/database".format(CRUFTSPACE)
CERTIFICATE_DIR = "{}/certs".format(CRUFTSPACE)
def spin_up_ursula(rest_port, db_name, teachers=(), certificate_dir=None):
metadata_file = "examples-runtime-cruft/node-metadata-{}".format(rest_port)
asyncio.set_event_loop(asyncio.new_event_loop()) # Ugh. Awful. But needed until we shed the DHT.
_URSULA = Ursula(rest_port=rest_port,
rest_host="localhost",
db_name=db_name,
federated_only=True,
known_nodes=teachers,
known_certificates_dir=certificate_dir
)
try:
with open(metadata_file, "w") as f:
f.write(bytes(_URSULA).hex())
_URSULA.start_learning_loop()
_URSULA.get_deployer().run()
finally:
os.remove(db_name)
os.remove(metadata_file)
if __name__ == "__main__":
try:
shutil.rmtree(CRUFTSPACE, ignore_errors=True)
os.mkdir(CRUFTSPACE)
os.mkdir(CERTIFICATE_DIR)
try:
teacher_rest_port = sys.argv[2]
# TODO: Implement real path tooling here.
with open("{}/node-metadata-{}".format(SHARED_CRUFTSPACE,
teacher_rest_port), "r") as f:
f.seek(0)
teacher_bytes = binascii.unhexlify(f.read())
teacher = Ursula.from_bytes(teacher_bytes,
federated_only=True)
teacher.save_certificate_to_disk(directory=CERTIFICATE_DIR)
teachers = (teacher,)
print("Will learn from {}".format(teacher))
except IndexError:
teachers = ()
except FileNotFoundError as e:
raise ValueError("Can't find a metadata file for node {}".format(teacher_rest_port))
spin_up_ursula(MY_REST_PORT, DB_NAME,
teachers=teachers,
certificate_dir=CERTIFICATE_DIR)
finally:
shutil.rmtree(CRUFTSPACE)

View File

@ -497,12 +497,13 @@ class Character(Learner):
nodes_i_need=self._node_ids_to_learn_about_immediately,
announce_nodes=announce_nodes,
certificate_path=current_teacher.certificate_filepath)
except requests.exceptions.ConnectionError:
except requests.exceptions.ConnectionError as e:
unresponsive_nodes.add(current_teacher)
teacher_rest_info = current_teacher.rest_information()[0]
# TODO: This error isn't necessarily "no repsonse" - let's maybe pass on the text of the exception here.
self.log.info("No Response from teacher: {}:{}.".format(teacher_rest_info.host, teacher_rest_info.port))
self.cycle_teacher_node()
self.learn_from_teacher_node()
return
if response.status_code != 200:
@ -543,6 +544,11 @@ class Character(Learner):
current_teacher.checksum_public_address,
len(node_list),
len(new_nodes)), )
if new_nodes and self.known_certificates_dir:
for node in new_nodes:
node.save_certificate_to_disk(self.known_certificates_dir)
return new_nodes
def encrypt_for(self,
recipient: 'Character',

View File

@ -468,6 +468,7 @@ class Ursula(Character, VerifiableNode, Miner):
stamp=self.stamp,
verifier=self.verify_from,
suspicious_activity_tracker=self.suspicious_activities_witnessed,
certificate_dir=self.known_certificates_dir,
)
rest_server = ProxyRESTServer(
@ -498,7 +499,9 @@ class Ursula(Character, VerifiableNode, Miner):
if certificate or certificate_filepath:
tls_hosting_power = TLSHostingPower(rest_server=rest_server,
certificate_filepath=certificate_filepath,
certificate=certificate)
certificate=certificate,
certificate_dir=self.known_certificates_dir,
common_name=self.checksum_public_address,)
else:
tls_hosting_keypair = HostingKeypair(
common_name=self.checksum_public_address,
@ -580,7 +583,8 @@ class Ursula(Character, VerifiableNode, Miner):
@classmethod
def from_bytes(cls, ursula_as_bytes: bytes,
federated_only: bool = False) -> 'Ursula':
federated_only: bool = False,
) -> 'Ursula':
(signature,
identity_evidence,
@ -598,14 +602,15 @@ class Ursula(Character, VerifiableNode, Miner):
rest_host=rest_info.host,
rest_port=rest_info.port,
certificate=certificate,
federated_only=federated_only # TODO: 289
federated_only=federated_only, # TODO: 289
)
return stranger_ursula_from_public_keys
@classmethod
def batch_from_bytes(cls,
ursulas_as_bytes: Iterable[bytes],
federated_only: bool = False) -> List['Ursula']:
federated_only: bool = False,
) -> List['Ursula']:
# TODO: Make a better splitter for this. This is a workaround until bytestringSplitter #8 is closed.

View File

@ -115,22 +115,16 @@ def ecdsa_verify(message: bytes,
def _save_tls_certificate(certificate: Certificate,
directory: str,
common_name: str = None,
full_filepath,
force: bool = True,
) -> str:
if force is False and os.path.isfile(full_filepath):
raise FileExistsError('A TLS certificate already exists at {}.'.format(full_filepath))
certificate_filepath = os.path.join(directory, '{}.pem'.format(common_name[2:8]))
if force is False and os.path.isfile(certificate_filepath):
raise FileExistsError('A TLS certificate already exists at {}.'.format(certificate_filepath))
with open(certificate_filepath, 'wb') as certificate_file:
with open(full_filepath, 'wb') as certificate_file:
public_pem_bytes = certificate.public_bytes(Encoding.PEM)
certificate_file.write(public_pem_bytes)
return certificate_filepath
def load_tls_certificate(filepath: str) -> Certificate:
"""Deserialize an X509 certificate from a filepath"""
@ -170,12 +164,7 @@ def generate_self_signed_certificate(common_name: str,
cert = cert.add_extension(x509.SubjectAlternativeName([x509.DNSName(host)]), critical=False)
cert = cert.sign(private_key, hashes.SHA512(), default_backend())
if certificate_dir: # TODO: Make this more configurable?
tls_certificate_filepath = _save_tls_certificate(cert, directory=certificate_dir, common_name=common_name)
else:
tls_certificate_filepath = constants.CERTIFICATE_NOT_SAVED
return cert, private_key, tls_certificate_filepath
return cert, private_key
def encrypt_and_sign(recipient_pubkey_enc: UmbralPublicKey,

View File

@ -13,7 +13,7 @@ from umbral.keys import UmbralPrivateKey, UmbralPublicKey
from umbral.signing import Signature, Signer
from nucypher.crypto import api as API
from nucypher.crypto.api import generate_self_signed_certificate, load_tls_certificate
from nucypher.crypto.api import generate_self_signed_certificate, load_tls_certificate, _save_tls_certificate
from nucypher.crypto.kits import MessageKit
from nucypher.crypto.signing import SignatureStamp, StrangerStamp
@ -157,7 +157,7 @@ class HostingKeypair(Keypair):
"But for that, you need to pass both host and common_name.."
raise TypeError(message)
certificate, private_key, certificate_filepath = generate_self_signed_certificate(common_name=common_name,
certificate, private_key = generate_self_signed_certificate(common_name=common_name,
private_key=private_key,
curve=self.curve,
host=host,
@ -167,6 +167,9 @@ class HostingKeypair(Keypair):
else:
raise TypeError("You didn't provide a cert, but also told us not to generate keys. Not sure what to do.")
if not certificate_filepath:
certificate_filepath = constants.CERTIFICATE_NOT_SAVED
self.certificate = certificate
self.certificate_filepath = certificate_filepath
self.certificate_dir = certificate_dir

View File

@ -2,25 +2,22 @@ import requests
from bytestring_splitter import BytestringSplitter, VariableLengthBytestring
from nucypher.crypto.api import load_tls_certificate
from umbral.fragments import CapsuleFrag
class RestMiddleware:
def consider_arrangement(self, arrangement, certificate_path):
def consider_arrangement(self, arrangement):
node = arrangement.ursula
port = node.rest_interface.port
address = node.rest_interface.host
response = requests.post("https://{}:{}/consider_arrangement".format(address, port), bytes(arrangement), verify=certificate_path)
response = requests.post("https://{}/consider_arrangement".format(node.rest_interface), bytes(arrangement),
verify=arrangement.ursula.certificate_filepath)
if not response.status_code == 200:
raise RuntimeError("Bad response: {}".format(response.content))
return response
def enact_policy(self, ursula, id, payload, certificate_path):
port = ursula.rest_interface.port
address = ursula.rest_interface.host
response = requests.post('https://{}:{}/kFrag/{}'.format(address, port, id.hex()), payload, verify=certificate_path)
def enact_policy(self, ursula, id, payload):
response = requests.post('https://{}/kFrag/{}'.format(ursula.rest_interface, id.hex()), payload,
verify=ursula.certificate_filepath)
if not response.status_code == 200:
raise RuntimeError("Bad response: {}".format(response.content))
return True, ursula.stamp.as_umbral_pubkey()
@ -34,25 +31,21 @@ class RestMiddleware:
def get_competitive_rate(self):
return NotImplemented
def get_treasure_map_from_node(self, node, map_id, certificate_path):
port = node.rest_interface.port
address = node.rest_interface.host
endpoint = "https://{}:{}/treasure_map/{}".format(address, port, map_id)
response = requests.get(endpoint, verify=certificate_path)
def get_treasure_map_from_node(self, node, map_id):
endpoint = "https://{}/treasure_map/{}".format(node.rest_interface, map_id)
response = requests.get(endpoint, verify=node.certificate_filepath)
return response
def put_treasure_map_on_node(self, node, map_id, map_payload, certificate_path):
port = node.rest_interface.port
address = node.rest_interface.host
endpoint = "https://{}:{}/treasure_map/{}".format(address, port, map_id)
response = requests.post(endpoint, data=map_payload, verify=certificate_path)
def put_treasure_map_on_node(self, node, map_id, map_payload):
endpoint = "https://{}/treasure_map/{}".format(node.rest_interface, map_id)
response = requests.post(endpoint, data=map_payload, verify=node.certificate_filepath)
return response
def send_work_order_payload_to_ursula(self, work_order):
payload = work_order.payload()
id_as_hex = work_order.arrangement_id.hex()
endpoint = 'https://{}/kFrag/{}/reencrypt'.format(work_order.ursula.rest_url(), id_as_hex)
return requests.post(endpoint, payload, verify=work_order.ursula.certificate_path)
endpoint = 'https://{}/kFrag/{}/reencrypt'.format(work_order.ursula.rest_interface, id_as_hex)
return requests.post(endpoint, payload, verify=work_order.ursula.certificate_filepath)
def node_information(self, host, port, certificate_path=None):
endpoint = "https://{}:{}/public_information".format(host, port)
@ -75,6 +68,6 @@ class RestMiddleware:
verify=certificate_path,
data=payload)
else:
response = requests.get("https://{}node_metadata".format(url),
response = requests.get("https://{}/node_metadata".format(url),
verify=certificate_path)
return response

View File

@ -1,8 +1,11 @@
import OpenSSL
from constant_sorrow import constants
from eth_keys.datatypes import Signature as EthSignature
from nucypher.crypto.api import _save_tls_certificate
from nucypher.crypto.powers import BlockchainPower, SigningPower, EncryptingPower, NoSigningPower
from nucypher.network.protocols import SuspiciousActivity
from nucypher.network.server import TLSHostingPower
from nucypher.utilities.sandbox.constants import TEST_URSULA_INSECURE_DEVELOPMENT_PASSWORD
@ -18,7 +21,7 @@ class VerifiableNode:
certificate_filepath: str = None,
) -> None:
self.certificate_filepath = certificate_filepath
self.certificate_filepath = certificate_filepath # TODO: This gets messy when it is None (although it being None is actually reasonable in some cases, at least for testing). Let's make this a method instead that inspects the TLSHostingPower (similar to get_deployer()).
self._interface_signature_object = interface_signature
class InvalidNode(SuspiciousActivity):
@ -137,3 +140,22 @@ class VerifiableNode:
except NoSigningPower:
raise NoSigningPower("This Ursula is a Stranger; you didn't init with an interface signature, so you can't verify.")
return self._interface_signature_object
def certificate(self):
return self._crypto_power.power_ups(TLSHostingPower).keypair.certificate
def save_certificate_to_disk(self, directory):
x509 = OpenSSL.crypto.X509.from_cryptography(self.certificate())
subject_components = x509.get_subject().get_components()
common_name_as_bytes = subject_components[0][1]
common_name_from_cert = common_name_as_bytes.decode()
if not self.checksum_public_address == common_name_from_cert:
# TODO: It's better for us to have checked this a while ago so that this situation is impossible. #443
raise ValueError(
"You passed a common_name that is not the same one as the cert. Why? FWIW, You don't even need to pass a common name here; the cert will be saved according to the name on the cert itself.")
certificate_filepath = "{}/{}".format(directory,
common_name_from_cert) # TODO: Do this with proper path tooling.
_save_tls_certificate(self.certificate(), full_filepath=certificate_filepath)
self.certificate_filepath = certificate_filepath

View File

@ -52,6 +52,7 @@ class ProxyRESTRoutes:
stamp,
verifier,
suspicious_activity_tracker,
certificate_dir,
) -> None:
self.network_middleware = network_middleware
@ -65,6 +66,7 @@ class ProxyRESTRoutes:
self._stamp = stamp
self._verifier = verifier
self._suspicious_activity_tracker = suspicious_activity_tracker
self._certificate_dir = certificate_dir
self.datastore = None
routes = [
@ -128,7 +130,9 @@ class ProxyRESTRoutes:
return Response(bytes(signature) + ursulas_as_bytes, headers=headers)
def node_metadata_exchange(self, request: http.Request, query_params: http.QueryParams):
nodes = self._node_class.batch_from_bytes(request.body, federated_only=self.federated_only)
nodes = self._node_class.batch_from_bytes(request.body,
federated_only=self.federated_only,
)
# TODO: This logic is basically repeated in learn_from_teacher_node. Let's find a better way.
for node in nodes:
@ -147,6 +151,8 @@ class ProxyRESTRoutes:
self._suspicious_activity_tracker['vladimirs'].append(node) # TODO: Maybe also record the bytes representation separately to disk?
else:
self.log.info("Previously unknown node: {}".format(node.checksum_public_address))
if self._certificate_dir:
node.save_certificate_to_disk(self._certificate_dir)
self._node_recorder(node)
# TODO: What's the right status code here? 202? Different if we already knew about the node?
@ -282,10 +288,18 @@ class TLSHostingPower(KeyPairBasedPower):
rest_server,
certificate_filepath=None,
certificate=None,
certificate_dir=None,
common_name=None, # TODO: Is this actually optional?
*args, **kwargs) -> None:
if certificate and certificate_filepath:
# TODO: Design decision here: if they do pass both, and they're identical, do we let that slide?
raise ValueError("Pass either a certificate or a certificate_filepath - what do you even expect from passing both?")
if certificate:
kwargs['keypair'] = HostingKeypair(certificate=certificate)
kwargs['keypair'] = HostingKeypair(certificate=certificate,
certificate_dir=certificate_dir,
common_name=common_name)
elif certificate_filepath:
kwargs['keypair'] = HostingKeypair(certificate_filepath=certificate_filepath)
self.rest_server = rest_server

View File

@ -47,8 +47,6 @@ class MockRestMiddleware(RestMiddleware):
mock_client = self._get_mock_client_by_ursula(work_order.ursula)
payload = work_order.payload()
id_as_hex = work_order.arrangement_id.hex()
assert os.path.exists(work_order.ursula.certificate_filepath), 'TLS Certificate does not exist on the filesystem'
return mock_client.post('http://localhost/kFrag/{}/reencrypt'.format(id_as_hex), payload)
def get_treasure_map_from_node(self, node, map_id):