From eba53f32065ec890dcbbd4c103fc86861ce5203e Mon Sep 17 00:00:00 2001 From: "Kieran R. Prasch" Date: Mon, 9 Dec 2019 23:07:16 -0800 Subject: [PATCH 1/3] Prepare datastore models for storing WorkOrders --- nucypher/characters/lawful.py | 7 ++-- nucypher/crypto/utils.py | 4 +- nucypher/keystore/db/models.py | 1 + nucypher/keystore/keystore.py | 49 +++++++++++++++------- nucypher/network/middleware.py | 5 +-- nucypher/network/server.py | 41 ++++++------------ nucypher/policy/collections.py | 3 +- tests/characters/test_bob_handles_frags.py | 2 +- tests/keystore/test_keystore.py | 6 +-- 9 files changed, 62 insertions(+), 56 deletions(-) diff --git a/nucypher/characters/lawful.py b/nucypher/characters/lawful.py index a16833786..373f7ee43 100644 --- a/nucypher/characters/lawful.py +++ b/nucypher/characters/lawful.py @@ -17,7 +17,6 @@ along with nucypher. If not, see . import json -import time from base64 import b64encode from collections import OrderedDict from functools import partial @@ -26,6 +25,7 @@ from typing import Dict, Iterable, List, Set, Tuple, Union import maya import requests +import time from bytestring_splitter import BytestringKwargifier, BytestringSplittingError from bytestring_splitter import BytestringSplitter, VariableLengthBytestring from constant_sorrow import constants @@ -38,7 +38,9 @@ from eth_utils import to_checksum_address from flask import request, Response from twisted.internet import threads from twisted.logger import Logger +from umbral import pre from umbral.keys import UmbralPublicKey +from umbral.kfrags import KFrag from umbral.pre import UmbralCorrectnessError from umbral.signing import Signature @@ -62,13 +64,13 @@ from nucypher.crypto.kits import UmbralMessageKit from nucypher.crypto.powers import SigningPower, DecryptingPower, DelegatingPower, TransactingPower, PowerUpError from nucypher.crypto.signing import InvalidSignature from nucypher.keystore.keypairs import HostingKeypair +from nucypher.keystore.threading import ThreadedSession from nucypher.network.exceptions import NodeSeemsToBeDown from nucypher.network.middleware import RestMiddleware, UnexpectedResponse, NotFound from nucypher.network.nicknames import nickname_from_seed from nucypher.network.nodes import Teacher from nucypher.network.protocols import InterfaceInfo, parse_node_uri from nucypher.network.server import ProxyRESTServer, TLSHostingPower, make_rest_app -from nucypher.blockchain.eth.decorators import validate_checksum_address class Alice(Character, BlockchainPolicyAuthor): @@ -820,7 +822,6 @@ class Ursula(Teacher, Character, Worker): from nucypher.config.node import CharacterConfiguration domains = (CharacterConfiguration.DEFAULT_DOMAIN,) - self._work_orders = list() Character.__init__(self, is_me=is_me, checksum_address=checksum_address, diff --git a/nucypher/crypto/utils.py b/nucypher/crypto/utils.py index aae26a112..d31c37446 100644 --- a/nucypher/crypto/utils.py +++ b/nucypher/crypto/utils.py @@ -15,10 +15,10 @@ You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ -from coincurve import PublicKey -from eth_keys import KeyAPI as EthKeyAPI from typing import Any, Union +from coincurve import PublicKey +from eth_keys import KeyAPI as EthKeyAPI from umbral.keys import UmbralPublicKey from umbral.point import Point from umbral.signing import Signature diff --git a/nucypher/keystore/db/models.py b/nucypher/keystore/db/models.py index 00623b842..0f10a493c 100644 --- a/nucypher/keystore/db/models.py +++ b/nucypher/keystore/db/models.py @@ -15,6 +15,7 @@ You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ from datetime import datetime + from sqlalchemy import ( Column, Integer, LargeBinary, ForeignKey, Boolean, DateTime ) diff --git a/nucypher/keystore/keystore.py b/nucypher/keystore/keystore.py index f21009581..e3627adfe 100644 --- a/nucypher/keystore/keystore.py +++ b/nucypher/keystore/keystore.py @@ -14,11 +14,12 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ +from typing import Union, List + from bytestring_splitter import BytestringSplitter from sqlalchemy.orm import sessionmaker -from typing import Union -from umbral.kfrags import KFrag from umbral.keys import UmbralPublicKey +from umbral.kfrags import KFrag from nucypher.crypto.signing import Signature from nucypher.crypto.utils import fingerprint_from_key @@ -79,10 +80,8 @@ class KeyStore(object): session = session or self._session_on_init_thread key = session.query(Key).filter_by(fingerprint=fingerprint).first() - if not key: - raise NotFound( - "No key with fingerprint {} found.".format(fingerprint)) + raise NotFound("No key with fingerprint {} found.".format(fingerprint)) pubkey = UmbralPublicKey.from_bytes(key.key_data) return pubkey @@ -160,30 +159,52 @@ class KeyStore(object): policy_arrangement.kfrag = bytes(kfrag) session.commit() - def add_workorder(self, bob_verifying_key, bob_signature, arrangement_id, session=None) -> Workorder: + def save_workorder(self, bob_verifying_key, bob_signature, arrangement_id, session=None) -> Workorder: """ Adds a Workorder to the keystore. """ session = session or self._session_on_init_thread - bob_verifying_key = self.add_key(bob_verifying_key) - new_workorder = Workorder(bob_verifying_key.id, bob_signature, arrangement_id) + + # Get or Create Bob Verifying Key + fingerprint = fingerprint_from_key(bob_verifying_key) + key = session.query(Key).filter_by(fingerprint=fingerprint).first() + if not key: + key = self.add_key(key=bob_verifying_key) + + new_workorder = Workorder(bob_verifying_key_id=key.id, + bob_signature=bob_signature, + arrangement_id=arrangement_id) session.add(new_workorder) session.commit() - return new_workorder - def get_workorders(self, arrangement_id: bytes, session=None) -> Workorder: + def get_workorders(self, + arrangement_id: bytes = None, + bob_verifying_key: bytes = None, + session=None + ) -> List[Workorder]: """ Returns a list of Workorders by HRAC. """ session = session or self._session_on_init_thread + query = session.query(Workorder) - workorders = session.query(Workorder).filter_by(arrangement_id=arrangement_id) + if not arrangement_id and not bob_verifying_key: + workorders = list(query.all()) + else: + if arrangement_id: + workorders = query.filter_by(arrangement_id=arrangement_id) + elif bob_verifying_key: + fingerprint = fingerprint_from_key(bob_verifying_key) + bvk = session.query(Key).filter_by(fingerprint=fingerprint).first() + workorders = query.filter_by(bob_verifying_key_id=bvk.id) + else: + raise ValueError + if not workorders: + raise NotFound - if not workorders: - raise NotFound("No Workorders with {} HRAC found.".format(arrangement_id)) - return workorders + return list(workorders) def del_workorders(self, arrangement_id: bytes, session=None): """ diff --git a/nucypher/network/middleware.py b/nucypher/network/middleware.py index 7e7bcf094..ad0eae509 100644 --- a/nucypher/network/middleware.py +++ b/nucypher/network/middleware.py @@ -19,14 +19,13 @@ import ssl import requests import time +from bytestring_splitter import BytestringSplitter, VariableLengthBytestring +from constant_sorrow.constants import CERTIFICATE_NOT_SAVED from cryptography import x509 from cryptography.hazmat.backends import default_backend from twisted.logger import Logger from umbral.cfrags import CapsuleFrag from umbral.signing import Signature -from constant_sorrow.constants import CERTIFICATE_NOT_SAVED - -from bytestring_splitter import BytestringSplitter, VariableLengthBytestring class UnexpectedResponse(Exception): diff --git a/nucypher/network/server.py b/nucypher/network/server.py index ca48f76cd..ea2ce8b5e 100644 --- a/nucypher/network/server.py +++ b/nucypher/network/server.py @@ -19,18 +19,16 @@ import binascii import os from typing import Tuple -from flask import Flask, Response -from flask import request -from jinja2 import Template, TemplateError -from twisted.logger import Logger -from umbral import pre -from umbral.keys import UmbralPublicKey -from umbral.kfrags import KFrag - from bytestring_splitter import VariableLengthBytestring from constant_sorrow import constants from constant_sorrow.constants import FLEET_STATES_MATCH, NO_KNOWN_NODES +from flask import Flask, Response +from flask import request from hendrix.experience import crosstown_traffic +from jinja2 import Template, TemplateError +from twisted.logger import Logger +from umbral.keys import UmbralPublicKey +from umbral.kfrags import KFrag import nucypher from nucypher.config.storages import ForgetfulNodeStorage @@ -297,8 +295,7 @@ def make_rest_app( arrangement_id = binascii.unhexlify(id_as_hex) try: with ThreadedSession(db_engine) as session: - policy_arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), - session=session) + arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session) except NotFound: return Response(response=arrangement_id, status=404) kfrag_bytes = policy_arrangement.kfrag # Careful! :-) @@ -318,26 +315,14 @@ def make_rest_app( cfrag_byte_stream = b"" - for task in work_order.tasks: - # Ursula signs on top of Bob's signature of each task. - # Now both are committed to the same task. See #259. - reencryption_metadata = bytes(this_node.stamp(bytes(task.signature))) - - capsule = task.capsule - capsule.set_correctness_keys(verifying=alices_verifying_key) - cfrag = pre.reencrypt(kfrag, capsule, metadata=reencryption_metadata) - log.info(f"Re-encrypting for {capsule}, made {cfrag}.") - - # Finally, Ursula commits to her result - reencryption_signature = this_node.stamp(bytes(cfrag)) - cfrag_byte_stream += VariableLengthBytestring(cfrag) + reencryption_signature - - # TODO: Put this in Ursula's datastore - this_node._work_orders.append(work_order) + # Now, Ursula saves this workorder to her database... + with ThreadedSession(db_engine): + this_node.datastore.save_workorder(bob_verifying_key=bytes(work_order.bob.stamp), + bob_signature=bytes(work_order.receipt_signature), + arrangement_id=work_order.arrangement_id) headers = {'Content-Type': 'application/octet-stream'} - - return Response(response=cfrag_byte_stream, headers=headers) + return Response(headers=headers, response=response) @rest_app.route('/treasure_map/') def provide_treasure_map(treasure_map_id): diff --git a/nucypher/policy/collections.py b/nucypher/policy/collections.py index 3dff04616..2b9d7121f 100644 --- a/nucypher/policy/collections.py +++ b/nucypher/policy/collections.py @@ -15,6 +15,7 @@ You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ import binascii +from typing import List, Optional, Tuple import maya import msgpack @@ -23,8 +24,6 @@ from constant_sorrow.constants import NO_DECRYPTION_PERFORMED from cryptography.hazmat.backends.openssl import backend from cryptography.hazmat.primitives import hashes from eth_utils import to_canonical_address, to_checksum_address -from typing import List, Optional, Tuple - from umbral.cfrags import CapsuleFrag from umbral.config import default_params from umbral.curvebn import CurveBN diff --git a/tests/characters/test_bob_handles_frags.py b/tests/characters/test_bob_handles_frags.py index 20a7c315e..b9cb5c40c 100644 --- a/tests/characters/test_bob_handles_frags.py +++ b/tests/characters/test_bob_handles_frags.py @@ -215,7 +215,7 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_federated_polic # Now we'll show that Ursula saved the correct WorkOrder. work_orders_from_bob = ursula.work_orders(bob=federated_bob) assert len(work_orders_from_bob) == 1 - assert work_orders_from_bob[0] == work_order + assert work_orders_from_bob[0].bob_signature == work_order.receipt_signature def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_federated_policy, federated_bob, diff --git a/tests/keystore/test_keystore.py b/tests/keystore/test_keystore.py index 321423316..524bbe56e 100644 --- a/tests/keystore/test_keystore.py +++ b/tests/keystore/test_keystore.py @@ -64,8 +64,8 @@ def test_workorder_sqlite_keystore(test_keystore): arrangement_id = b'test' # Test add workorder - new_workorder1 = test_keystore.add_workorder(bob_keypair_sig1.pubkey, b'test0', arrangement_id) - new_workorder2 = test_keystore.add_workorder(bob_keypair_sig2.pubkey, b'test1', arrangement_id) + new_workorder1 = test_keystore.save_workorder(bob_keypair_sig1.pubkey, b'test0', arrangement_id) + new_workorder2 = test_keystore.save_workorder(bob_keypair_sig2.pubkey, b'test1', arrangement_id) # Test get workorder query_workorders = test_keystore.get_workorders(arrangement_id) @@ -74,4 +74,4 @@ def test_workorder_sqlite_keystore(test_keystore): # Test del workorder deleted = test_keystore.del_workorders(arrangement_id) assert deleted > 0 - assert test_keystore.get_workorders(arrangement_id).count() == 0 + assert test_keystore.get_workorders(arrangement_id) == 0 From 77b297ede3745dfe5fa5c4fbdb85a9be0121d02d Mon Sep 17 00:00:00 2001 From: "Kieran R. Prasch" Date: Mon, 9 Dec 2019 23:07:49 -0800 Subject: [PATCH 2/3] Excavate reencryption logic onto Ursula proper. --- nucypher/characters/lawful.py | 47 ++++++++++++++++++++++++--------- nucypher/cli/painting.py | 2 +- nucypher/network/server.py | 32 +++++++++++++--------- tests/keystore/test_keystore.py | 2 +- 4 files changed, 56 insertions(+), 27 deletions(-) diff --git a/nucypher/characters/lawful.py b/nucypher/characters/lawful.py index 373f7ee43..443a131e6 100644 --- a/nucypher/characters/lawful.py +++ b/nucypher/characters/lawful.py @@ -1252,21 +1252,42 @@ class Ursula(Teacher, Character, Worker): return constants.BYTESTRING_IS_URSULA_IFACE_INFO + bytes(self) # - # Utilities + # Work Orders & Re-Encryption # - def work_orders(self, bob=None): - """ - TODO: This is better written as a model method for Ursula's datastore. - """ - if not bob: - return self._work_orders - else: - work_orders_from_bob = [] - for work_order in self._work_orders: - if work_order.bob == bob: - work_orders_from_bob.append(work_order) - return work_orders_from_bob + def work_orders(self, bob=None) -> List['WorkOrder']: + with ThreadedSession(self.datastore.engine): + if not bob: # All + return self.datastore.get_workorders() + else: # Filter + work_orders_from_bob = self.datastore.get_workorders(bob_verifying_key=bytes(bob.stamp)) + return work_orders_from_bob + + def _reencrypt(self, kfrag: KFrag, work_order: 'WorkOrder', alice_verifying_key: UmbralPublicKey): + + # Prepare a bytestring for concatenating re-encrypted + # capsule data for each work order task. + cfrag_byte_stream = bytes() + for task in work_order.tasks: + + # Ursula signs on top of Bob's signature of each task. + # Now both are committed to the same task. See #259. + reencryption_metadata = bytes(self.stamp(bytes(task.signature))) + + # Ursula sets Alice's verifying key for capsule correctness verification. + capsule = task.capsule + capsule.set_correctness_keys(verifying=alice_verifying_key) + + # Then re-encrypts the fragment. + cfrag = pre.reencrypt(kfrag, capsule, metadata=reencryption_metadata) # <--- pyUmbral + self.log.info(f"Re-encrypted capsule {capsule} -> made {cfrag}.") + + # Next, Ursula signs to commit to her results. + reencryption_signature = self.stamp(bytes(cfrag)) + cfrag_byte_stream += VariableLengthBytestring(cfrag) + reencryption_signature + + # ... and finally returns all the re-encrypted bytes + return cfrag_byte_stream class Enrico(Character): diff --git a/nucypher/cli/painting.py b/nucypher/cli/painting.py index 72cc6a1db..e6b75ddeb 100644 --- a/nucypher/cli/painting.py +++ b/nucypher/cli/painting.py @@ -124,7 +124,7 @@ def paint_node_status(emitter, ursula, start_time): 'Rest Interface ...... {}'.format(ursula.rest_url()), 'Node Storage Type ... {}'.format(ursula.node_storage._name.capitalize()), 'Known Nodes ......... {}'.format(len(ursula.known_nodes)), - 'Work Orders ......... {}'.format(len(ursula._work_orders)), + 'Work Orders ......... {}'.format(len(ursula.work_orders())), teacher] if not ursula.federated_only: diff --git a/nucypher/network/server.py b/nucypher/network/server.py index ea2ce8b5e..849f6eb7c 100644 --- a/nucypher/network/server.py +++ b/nucypher/network/server.py @@ -291,29 +291,37 @@ def make_rest_app( @rest_app.route('/kFrag//reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): - from nucypher.policy.collections import WorkOrder # Avoid circular import - arrangement_id = binascii.unhexlify(id_as_hex) + + # Get Policy Arrangement + try: + arrangement_id = binascii.unhexlify(id_as_hex) + except (binascii.Error, TypeError): + return Response(response=b'Invalid arrangement ID', status=405) try: with ThreadedSession(db_engine) as session: arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session) except NotFound: return Response(response=arrangement_id, status=404) - kfrag_bytes = policy_arrangement.kfrag # Careful! :-) - verifying_key_bytes = policy_arrangement.alice_verifying_key.key_data - # TODO: Push this to a lower level. Perhaps to Ursula character? #619 - kfrag = KFrag.from_bytes(kfrag_bytes) - alices_verifying_key = UmbralPublicKey.from_bytes(verifying_key_bytes) - alices_address = canonical_address_from_umbral_key(alices_verifying_key) + # Get KFrag + kfrag = KFrag.from_bytes(arrangement.kfrag) # Careful! :-) + # Get Work Order + from nucypher.policy.collections import WorkOrder # Avoid circular import + alice_verifying_key_bytes = arrangement.alice_verifying_key.key_data + alice_verifying_key = UmbralPublicKey.from_bytes(alice_verifying_key_bytes) + alice_address = canonical_address_from_umbral_key(alice_verifying_key) + work_order_payload = request.data work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id, - rest_payload=request.data, + rest_payload=work_order_payload, ursula=this_node, - alice_address=alices_address) - + alice_address=alice_address) log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}") - cfrag_byte_stream = b"" + # Re-encrypt + response = this_node._reencrypt(kfrag=kfrag, + work_order=work_order, + alice_verifying_key=alice_verifying_key) # Now, Ursula saves this workorder to her database... with ThreadedSession(db_engine): diff --git a/tests/keystore/test_keystore.py b/tests/keystore/test_keystore.py index 524bbe56e..a8b3247f3 100644 --- a/tests/keystore/test_keystore.py +++ b/tests/keystore/test_keystore.py @@ -74,4 +74,4 @@ def test_workorder_sqlite_keystore(test_keystore): # Test del workorder deleted = test_keystore.del_workorders(arrangement_id) assert deleted > 0 - assert test_keystore.get_workorders(arrangement_id) == 0 + assert len(test_keystore.get_workorders(arrangement_id)) == 0 From fd2bc932f6a32a1abd38dff0f302244c3ddcc4ac Mon Sep 17 00:00:00 2001 From: "Kieran R. Prasch" Date: Fri, 13 Dec 2019 10:02:05 -0800 Subject: [PATCH 3/3] Respond to RFCs in PR #1523 --- nucypher/keystore/keystore.py | 16 ++++++++++------ nucypher/network/server.py | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/nucypher/keystore/keystore.py b/nucypher/keystore/keystore.py index e3627adfe..134796abd 100644 --- a/nucypher/keystore/keystore.py +++ b/nucypher/keystore/keystore.py @@ -191,16 +191,20 @@ class KeyStore(object): query = session.query(Workorder) if not arrangement_id and not bob_verifying_key: - workorders = list(query.all()) + workorders = query.all() # Return all records + else: + + # Return arrangement records if arrangement_id: workorders = query.filter_by(arrangement_id=arrangement_id) - elif bob_verifying_key: - fingerprint = fingerprint_from_key(bob_verifying_key) - bvk = session.query(Key).filter_by(fingerprint=fingerprint).first() - workorders = query.filter_by(bob_verifying_key_id=bvk.id) + + # Return records for Bob else: - raise ValueError + fingerprint = fingerprint_from_key(bob_verifying_key) + key = session.query(Key).filter_by(fingerprint=fingerprint).first() + workorders = query.filter_by(bob_verifying_key_id=key.id) + if not workorders: raise NotFound diff --git a/nucypher/network/server.py b/nucypher/network/server.py index 849f6eb7c..be24a472b 100644 --- a/nucypher/network/server.py +++ b/nucypher/network/server.py @@ -304,7 +304,7 @@ def make_rest_app( return Response(response=arrangement_id, status=404) # Get KFrag - kfrag = KFrag.from_bytes(arrangement.kfrag) # Careful! :-) + kfrag = KFrag.from_bytes(arrangement.kfrag) # Get Work Order from nucypher.policy.collections import WorkOrder # Avoid circular import