Merge pull request #1523 from KPrasch/carlin

Excavate Re-encryption logic; WorkOrder Datastore Models

nucypher/characters/lawful.py

@@ -17,7 +17,6 @@ along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
 
 
 import json
-import time
 from base64 import b64encode
 from collections import OrderedDict
 from functools import partial
@@ -26,6 +25,7 @@ from typing import Dict, Iterable, List, Set, Tuple, Union
 
 import maya
 import requests
+import time
 from bytestring_splitter import BytestringKwargifier, BytestringSplittingError
 from bytestring_splitter import BytestringSplitter, VariableLengthBytestring
 from constant_sorrow import constants
@@ -38,7 +38,9 @@ from eth_utils import to_checksum_address
 from flask import request, Response
 from twisted.internet import threads
 from twisted.logger import Logger
+from umbral import pre
 from umbral.keys import UmbralPublicKey
+from umbral.kfrags import KFrag
 from umbral.pre import UmbralCorrectnessError
 from umbral.signing import Signature
 
@@ -62,13 +64,13 @@ from nucypher.crypto.kits import UmbralMessageKit
 from nucypher.crypto.powers import SigningPower, DecryptingPower, DelegatingPower, TransactingPower, PowerUpError
 from nucypher.crypto.signing import InvalidSignature
 from nucypher.keystore.keypairs import HostingKeypair
+from nucypher.keystore.threading import ThreadedSession
 from nucypher.network.exceptions import NodeSeemsToBeDown
 from nucypher.network.middleware import RestMiddleware, UnexpectedResponse, NotFound
 from nucypher.network.nicknames import nickname_from_seed
 from nucypher.network.nodes import Teacher
 from nucypher.network.protocols import InterfaceInfo, parse_node_uri
 from nucypher.network.server import ProxyRESTServer, TLSHostingPower, make_rest_app
-from nucypher.blockchain.eth.decorators import validate_checksum_address
 
 
 class Alice(Character, BlockchainPolicyAuthor):
@@ -820,7 +822,6 @@ class Ursula(Teacher, Character, Worker):
             from nucypher.config.node import CharacterConfiguration
             domains = (CharacterConfiguration.DEFAULT_DOMAIN,)
 
-        self._work_orders = list()
         Character.__init__(self,
                            is_me=is_me,
                            checksum_address=checksum_address,
@@ -1251,22 +1252,43 @@ class Ursula(Teacher, Character, Worker):
         return constants.BYTESTRING_IS_URSULA_IFACE_INFO + bytes(self)
 
     #
-    # Utilities
+    # Work Orders & Re-Encryption
     #
 
-    def work_orders(self, bob=None):
+    def work_orders(self, bob=None) -> List['WorkOrder']:
-        """
+        with ThreadedSession(self.datastore.engine):
-        TODO: This is better written as a model method for Ursula's datastore.
+            if not bob:  # All
-        """
+                return self.datastore.get_workorders()
-        if not bob:
+            else:        # Filter
-            return self._work_orders
+                work_orders_from_bob = self.datastore.get_workorders(bob_verifying_key=bytes(bob.stamp))
-        else:
-            work_orders_from_bob = []
-            for work_order in self._work_orders:
-                if work_order.bob == bob:
-                    work_orders_from_bob.append(work_order)
                 return work_orders_from_bob
 
+    def _reencrypt(self, kfrag: KFrag, work_order: 'WorkOrder', alice_verifying_key: UmbralPublicKey):
+        
+        # Prepare a bytestring for concatenating re-encrypted
+        # capsule data for each work order task.
+        cfrag_byte_stream = bytes()
+        for task in work_order.tasks:
+
+            # Ursula signs on top of Bob's signature of each task.
+            # Now both are committed to the same task.  See #259.
+            reencryption_metadata = bytes(self.stamp(bytes(task.signature)))
+
+            # Ursula sets Alice's verifying key for capsule correctness verification.
+            capsule = task.capsule
+            capsule.set_correctness_keys(verifying=alice_verifying_key)
+
+            # Then re-encrypts the fragment.
+            cfrag = pre.reencrypt(kfrag, capsule, metadata=reencryption_metadata)  # <--- pyUmbral
+            self.log.info(f"Re-encrypted capsule {capsule} -> made {cfrag}.")
+
+            # Next, Ursula signs to commit to her results.
+            reencryption_signature = self.stamp(bytes(cfrag))
+            cfrag_byte_stream += VariableLengthBytestring(cfrag) + reencryption_signature
+
+        # ... and finally returns all the re-encrypted bytes
+        return cfrag_byte_stream
+
 
 class Enrico(Character):
     """A Character that represents a Data Source that encrypts data for some policy's public key"""

nucypher/cli/painting.py

@@ -124,7 +124,7 @@ def paint_node_status(emitter, ursula, start_time):
              'Rest Interface ...... {}'.format(ursula.rest_url()),
              'Node Storage Type ... {}'.format(ursula.node_storage._name.capitalize()),
              'Known Nodes ......... {}'.format(len(ursula.known_nodes)),
-             'Work Orders ......... {}'.format(len(ursula._work_orders)),
+             'Work Orders ......... {}'.format(len(ursula.work_orders())),
              teacher]
 
     if not ursula.federated_only:

nucypher/crypto/utils.py

@@ -15,10 +15,10 @@ You should have received a copy of the GNU Affero General Public License
 along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-from coincurve import PublicKey
-from eth_keys import KeyAPI as EthKeyAPI
 from typing import Any, Union
 
+from coincurve import PublicKey
+from eth_keys import KeyAPI as EthKeyAPI
 from umbral.keys import UmbralPublicKey
 from umbral.point import Point
 from umbral.signing import Signature

nucypher/keystore/db/models.py

@@ -15,6 +15,7 @@ You should have received a copy of the GNU Affero General Public License
 along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
 """
 from datetime import datetime
+
 from sqlalchemy import (
     Column, Integer, LargeBinary, ForeignKey, Boolean, DateTime
 )

nucypher/keystore/keystore.py

@@ -14,11 +14,12 @@ GNU Affero General Public License for more details.
 You should have received a copy of the GNU Affero General Public License
 along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
 """
+from typing import Union, List
+
 from bytestring_splitter import BytestringSplitter
 from sqlalchemy.orm import sessionmaker
-from typing import Union
-from umbral.kfrags import KFrag
 from umbral.keys import UmbralPublicKey
+from umbral.kfrags import KFrag
 
 from nucypher.crypto.signing import Signature
 from nucypher.crypto.utils import fingerprint_from_key
@@ -79,10 +80,8 @@ class KeyStore(object):
         session = session or self._session_on_init_thread
 
         key = session.query(Key).filter_by(fingerprint=fingerprint).first()
-
         if not key:
-            raise NotFound(
+            raise NotFound("No key with fingerprint {} found.".format(fingerprint))
-                "No key with fingerprint {} found.".format(fingerprint))
 
         pubkey = UmbralPublicKey.from_bytes(key.key_data)
         return pubkey
@@ -160,30 +159,56 @@ class KeyStore(object):
         policy_arrangement.kfrag = bytes(kfrag)
         session.commit()
 
-    def add_workorder(self, bob_verifying_key, bob_signature, arrangement_id, session=None) -> Workorder:
+    def save_workorder(self, bob_verifying_key, bob_signature, arrangement_id, session=None) -> Workorder:
         """
         Adds a Workorder to the keystore.
         """
         session = session or self._session_on_init_thread
-        bob_verifying_key = self.add_key(bob_verifying_key)
+
-        new_workorder = Workorder(bob_verifying_key.id, bob_signature, arrangement_id)
+        # Get or Create Bob Verifying Key
+        fingerprint = fingerprint_from_key(bob_verifying_key)
+        key = session.query(Key).filter_by(fingerprint=fingerprint).first()
+        if not key:
+            key = self.add_key(key=bob_verifying_key)
+
+        new_workorder = Workorder(bob_verifying_key_id=key.id,
+                                  bob_signature=bob_signature,
+                                  arrangement_id=arrangement_id)
 
         session.add(new_workorder)
         session.commit()
-
         return new_workorder
 
-    def get_workorders(self, arrangement_id: bytes, session=None) -> Workorder:
+    def get_workorders(self,
+                       arrangement_id: bytes = None,
+                       bob_verifying_key: bytes = None,
+                       session=None
+                       ) -> List[Workorder]:
         """
         Returns a list of Workorders by HRAC.
         """
         session = session or self._session_on_init_thread
+        query = session.query(Workorder)
 
-        workorders = session.query(Workorder).filter_by(arrangement_id=arrangement_id)
+        if not arrangement_id and not bob_verifying_key:
+            workorders = query.all()  # Return all records
+
+        else:
+
+            # Return arrangement records
+            if arrangement_id:
+                workorders = query.filter_by(arrangement_id=arrangement_id)
+
+            # Return records for Bob
+            else:
+                fingerprint = fingerprint_from_key(bob_verifying_key)
+                key = session.query(Key).filter_by(fingerprint=fingerprint).first()
+                workorders = query.filter_by(bob_verifying_key_id=key.id)
 
             if not workorders:
-            raise NotFound("No Workorders with {} HRAC found.".format(arrangement_id))
+                raise NotFound
-        return workorders
+
+        return list(workorders)
 
     def del_workorders(self, arrangement_id: bytes, session=None):
         """

nucypher/network/middleware.py

@@ -19,14 +19,13 @@ import ssl
 
 import requests
 import time
+from bytestring_splitter import BytestringSplitter, VariableLengthBytestring
+from constant_sorrow.constants import CERTIFICATE_NOT_SAVED
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from twisted.logger import Logger
 from umbral.cfrags import CapsuleFrag
 from umbral.signing import Signature
-from constant_sorrow.constants import CERTIFICATE_NOT_SAVED
-
-from bytestring_splitter import BytestringSplitter, VariableLengthBytestring
 
 
 class UnexpectedResponse(Exception):

nucypher/network/server.py

@@ -19,18 +19,16 @@ import binascii
 import os
 from typing import Tuple
 
-from flask import Flask, Response
-from flask import request
-from jinja2 import Template, TemplateError
-from twisted.logger import Logger
-from umbral import pre
-from umbral.keys import UmbralPublicKey
-from umbral.kfrags import KFrag
-
 from bytestring_splitter import VariableLengthBytestring
 from constant_sorrow import constants
 from constant_sorrow.constants import FLEET_STATES_MATCH, NO_KNOWN_NODES
+from flask import Flask, Response
+from flask import request
 from hendrix.experience import crosstown_traffic
+from jinja2 import Template, TemplateError
+from twisted.logger import Logger
+from umbral.keys import UmbralPublicKey
+from umbral.kfrags import KFrag
 
 import nucypher
 from nucypher.config.storages import ForgetfulNodeStorage
@@ -293,51 +291,46 @@ def make_rest_app(
 
     @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"])
     def reencrypt_via_rest(id_as_hex):
-        from nucypher.policy.collections import WorkOrder  # Avoid circular import
+
+        # Get Policy Arrangement
+        try:
             arrangement_id = binascii.unhexlify(id_as_hex)
+        except (binascii.Error, TypeError):
+            return Response(response=b'Invalid arrangement ID', status=405)
         try:
             with ThreadedSession(db_engine) as session:
-                policy_arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(),
+                arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session)
-                                                                      session=session)
         except NotFound:
             return Response(response=arrangement_id, status=404)
-        kfrag_bytes = policy_arrangement.kfrag  # Careful!  :-)
-        verifying_key_bytes = policy_arrangement.alice_verifying_key.key_data
 
-        # TODO: Push this to a lower level. Perhaps to Ursula character? #619
+        # Get KFrag
-        kfrag = KFrag.from_bytes(kfrag_bytes)
+        kfrag = KFrag.from_bytes(arrangement.kfrag)
-        alices_verifying_key = UmbralPublicKey.from_bytes(verifying_key_bytes)
-        alices_address = canonical_address_from_umbral_key(alices_verifying_key)
 
+        # Get Work Order
+        from nucypher.policy.collections import WorkOrder  # Avoid circular import
+        alice_verifying_key_bytes = arrangement.alice_verifying_key.key_data
+        alice_verifying_key = UmbralPublicKey.from_bytes(alice_verifying_key_bytes)
+        alice_address = canonical_address_from_umbral_key(alice_verifying_key)
+        work_order_payload = request.data
         work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id,
-                                                 rest_payload=request.data,
+                                                 rest_payload=work_order_payload,
                                                  ursula=this_node,
-                                                 alice_address=alices_address)
+                                                 alice_address=alice_address)
-
         log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}")
 
-        cfrag_byte_stream = b""
+        # Re-encrypt
+        response = this_node._reencrypt(kfrag=kfrag,
+                                        work_order=work_order,
+                                        alice_verifying_key=alice_verifying_key)
 
-        for task in work_order.tasks:
+        # Now, Ursula saves this workorder to her database...
-            # Ursula signs on top of Bob's signature of each task.
+        with ThreadedSession(db_engine):
-            # Now both are committed to the same task.  See #259.
+            this_node.datastore.save_workorder(bob_verifying_key=bytes(work_order.bob.stamp),
-            reencryption_metadata = bytes(this_node.stamp(bytes(task.signature)))
+                                               bob_signature=bytes(work_order.receipt_signature),
-
+                                               arrangement_id=work_order.arrangement_id)
-            capsule = task.capsule
-            capsule.set_correctness_keys(verifying=alices_verifying_key)
-            cfrag = pre.reencrypt(kfrag, capsule, metadata=reencryption_metadata)
-            log.info(f"Re-encrypting for {capsule}, made {cfrag}.")
-
-            # Finally, Ursula commits to her result
-            reencryption_signature = this_node.stamp(bytes(cfrag))
-            cfrag_byte_stream += VariableLengthBytestring(cfrag) + reencryption_signature
-
-        # TODO: Put this in Ursula's datastore
-        this_node._work_orders.append(work_order)
 
         headers = {'Content-Type': 'application/octet-stream'}
-
+        return Response(headers=headers, response=response)
-        return Response(response=cfrag_byte_stream, headers=headers)
 
     @rest_app.route('/treasure_map/<treasure_map_id>')
     def provide_treasure_map(treasure_map_id):

nucypher/policy/collections.py

@@ -15,6 +15,7 @@ You should have received a copy of the GNU Affero General Public License
 along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
 """
 import binascii
+from typing import List, Optional, Tuple
 
 import maya
 import msgpack
@@ -23,8 +24,6 @@ from constant_sorrow.constants import NO_DECRYPTION_PERFORMED
 from cryptography.hazmat.backends.openssl import backend
 from cryptography.hazmat.primitives import hashes
 from eth_utils import to_canonical_address, to_checksum_address
-from typing import List, Optional, Tuple
-
 from umbral.cfrags import CapsuleFrag
 from umbral.config import default_params
 from umbral.curvebn import CurveBN

tests/characters/test_bob_handles_frags.py

@@ -215,7 +215,7 @@ def test_bob_can_issue_a_work_order_to_a_specific_ursula(enacted_federated_polic
     # Now we'll show that Ursula saved the correct WorkOrder.
     work_orders_from_bob = ursula.work_orders(bob=federated_bob)
     assert len(work_orders_from_bob) == 1
-    assert work_orders_from_bob[0] == work_order
+    assert work_orders_from_bob[0].bob_signature == work_order.receipt_signature
 
 
 def test_bob_remembers_that_he_has_cfrags_for_a_particular_capsule(enacted_federated_policy, federated_bob,

tests/keystore/test_keystore.py

@@ -64,8 +64,8 @@ def test_workorder_sqlite_keystore(test_keystore):
     arrangement_id = b'test'
 
     # Test add workorder
-    new_workorder1 = test_keystore.add_workorder(bob_keypair_sig1.pubkey, b'test0', arrangement_id)
+    new_workorder1 = test_keystore.save_workorder(bob_keypair_sig1.pubkey, b'test0', arrangement_id)
-    new_workorder2 = test_keystore.add_workorder(bob_keypair_sig2.pubkey, b'test1', arrangement_id)
+    new_workorder2 = test_keystore.save_workorder(bob_keypair_sig2.pubkey, b'test1', arrangement_id)
 
     # Test get workorder
     query_workorders = test_keystore.get_workorders(arrangement_id)
@@ -74,4 +74,4 @@ def test_workorder_sqlite_keystore(test_keystore):
     # Test del workorder
     deleted = test_keystore.del_workorders(arrangement_id)
     assert deleted > 0
-    assert test_keystore.get_workorders(arrangement_id).count() == 0
+    assert len(test_keystore.get_workorders(arrangement_id)) == 0