Add integration and acceptance tests for porter functionality.

Remove unused 'publish_treasure_map' parameter from _enact() function.
Loosen validation on treasure map id which can be 32 bytes (federated) or 16 bytes (non-federated). Probably only care about non-federated but for ease of testing, federated really helps.

nucypher/cli/commands/porter.py

@@ -112,7 +112,8 @@ def run(general_config, network, provider_uri, federated_only, teacher_uri, http
 
         BlockchainInterfaceFactory.initialize_interface(provider_uri=provider_uri)
         PORTER = Porter(domain=network,
-                        start_learning_now=eager)
+                        start_learning_now=eager,
+                        provider_uri=provider_uri)
 
     # RPC
     if general_config.json_ipc:

nucypher/policy/policies.py

@@ -515,7 +515,7 @@ class BlockchainPolicy(Policy):
 
     def _make_reservoir(self, handpicked_addresses):
         staker_reservoir = make_decentralized_staker_reservoir(staking_agent=self.alice.staking_agent,
-                                                               periods=self.payment_periods,
+                                                               duration_periods=self.payment_periods,
                                                                include_addresses=handpicked_addresses)
         return staker_reservoir
 

nucypher/policy/reservoir.py

@@ -21,7 +21,9 @@ from eth_typing import ChecksumAddress
 from nucypher.blockchain.eth.agents import StakersReservoir, StakingEscrowAgent
 
 
-def make_federated_staker_reservoir(learner: 'Learner', exclude_addresses=None, include_addresses=None):
+def make_federated_staker_reservoir(learner: 'Learner',
+                                    exclude_addresses: List[str] = None,
+                                    include_addresses: List[str] = None):
     """
     Get a sampler object containing the federated stakers.
     """
@@ -45,9 +47,9 @@ def make_federated_staker_reservoir(learner: 'Learner', exclude_addresses=None,
 
 
 def make_decentralized_staker_reservoir(staking_agent: StakingEscrowAgent,
-                                        periods,
-                                        exclude_addresses=None,
-                                        include_addresses=None):
+                                        duration_periods: int,
+                                        exclude_addresses: List[str] = None,
+                                        include_addresses: List[str] = None):
     """
     Get a sampler object containing the currently registered stakers.
     """
@@ -60,7 +62,7 @@ def make_decentralized_staker_reservoir(staking_agent: StakingEscrowAgent,
     without_set.update(exclude_addresses)
     without_set.update(include_addresses)
     try:
-        reservoir = staking_agent.get_stakers_reservoir(duration=periods,
+        reservoir = staking_agent.get_stakers_reservoir(duration=duration_periods,
                                                         without=without_set)
     except StakingEscrowAgent.NotEnoughStakers:
         # TODO: do that in `get_stakers_reservoir()`?

nucypher/utilities/porter/control/specifications/fields/treasuremapid.py

@@ -19,11 +19,14 @@ from marshmallow import fields
 
 from nucypher.control.specifications.exceptions import InvalidInputData
 from nucypher.control.specifications.fields.base import BaseField
+from nucypher.crypto.constants import HRAC_LENGTH
 from nucypher.policy.collections import TreasureMap
 
 
 class TreasureMapID(BaseField, fields.String):
 
     def _validate(self, value):
-        if len(bytes.fromhex(value)) != TreasureMap.ID_LENGTH:
+        treasure_map_id = bytes.fromhex(value)
+        # FIXME federated has map id length 32 bytes but decentralized has length 16 bytes ... huh?
+        if len(treasure_map_id) != TreasureMap.ID_LENGTH and len(treasure_map_id) != HRAC_LENGTH:
             raise InvalidInputData(f"Could not convert input for {self.name} to a valid TreasureMap ID: invalid length")

nucypher/utilities/porter/porter.py

@@ -21,6 +21,7 @@ from flask import request, Response
 from umbral.keys import UmbralPublicKey
 
 from nucypher.blockchain.eth.agents import ContractAgency, StakingEscrowAgent
+from nucypher.blockchain.eth.interfaces import BlockchainInterfaceFactory
 from nucypher.blockchain.eth.registry import BaseContractRegistry, InMemoryContractRegistry
 from nucypher.characters import utils
 from nucypher.characters.lawful import Ursula
@@ -64,16 +65,30 @@ the Pipe for nucypher network operations
 
     _interface_class = PorterInterface
 
+    class UrsulaInfo:
+        """Simple object that stores relevant Ursula information resulting from sampling."""
+
+        def __init__(self, checksum_address: str, ip_address: str, encrypting_key: UmbralPublicKey):
+            self.checksum_address = checksum_address
+            self.ip_address = ip_address
+            self.encrypting_key = encrypting_key
+
     def __init__(self,
                  domain: str = None,
                  registry: BaseContractRegistry = None,
                  controller: bool = True,
                  federated_only: bool = False,
                  node_class: object = Ursula,
+                 provider_uri: str = None,
                  *args, **kwargs):
         self.federated_only = federated_only
 
         if not self.federated_only:
+            if not provider_uri:
+                if not provider_uri:
+                    raise ValueError('Provider URI is required for decentralized Porter.')
+
+            BlockchainInterfaceFactory.get_interface(provider_uri=provider_uri)
             self.registry = registry or InMemoryContractRegistry.from_latest_publication(network=domain)
             self.staking_agent = ContractAgency.get_agent(StakingEscrowAgent, registry=self.registry)
         else:
@@ -110,7 +125,11 @@ the Pipe for nucypher network operations
         treasure_map_publisher.block_until_success_is_reasonably_likely()
         return
 
-    def get_ursulas(self, quantity: int, duration_periods: int, exclude_ursulas: List[str], include_ursulas: List[str]):
+    def get_ursulas(self,
+                    quantity: int,
+                    duration_periods: int,
+                    exclude_ursulas: List[str] = None,
+                    include_ursulas: List[str] = None) -> List[UrsulaInfo]:
         reservoir = self._make_staker_reservoir(quantity, duration_periods, exclude_ursulas, include_ursulas)
         value_factory = PrefetchStrategy(reservoir, quantity)
 
@@ -118,16 +137,16 @@ the Pipe for nucypher network operations
             if ursula_checksum not in self.known_nodes:
                 raise ValueError(f"{ursula_checksum} is not known")
 
-            # check connectivity
             ursula = self.known_nodes[ursula_checksum]
 
-            # don't care about the result only that it worked without raising an exception
+            # check connectivity - don't care about the result only that it worked without raising an exception
             # TODO is this the best way to check connectivity?
             _ = self.network_middleware.get_certificate(host=ursula.rest_interface.host,
                                                         port=ursula.rest_interface.port)
-            return UrsulaInfo(checksum_address=ursula_checksum,
-                              ip_address=f"https://{ursula.rest_interface.host}:{ursula.rest_interface.port}",
-                              encrypting_key=ursula.public_keys(DecryptingPower))
+
+            return Porter.UrsulaInfo(checksum_address=ursula_checksum,
+                                     ip_address=f"https://{ursula.rest_interface.host}:{ursula.rest_interface.port}",
+                                     encrypting_key=ursula.public_keys(DecryptingPower))
 
         self.block_until_number_of_known_nodes_is(quantity, learn_on_this_thread=True, eager=True)
 
@@ -138,27 +157,17 @@ the Pipe for nucypher network operations
                                  stagger_timeout=1,
                                  threadpool_size=quantity)
         worker_pool.start()
-        try:
-            successes = worker_pool.block_until_target_successes()
-        except (WorkerPool.OutOfValues, WorkerPool.TimedOut):
-            # It's possible to raise some other exceptions here,
-            # but we will use the logic below.
-            successes = worker_pool.get_successes()
-        finally:
-            worker_pool.cancel()
-            worker_pool.join()
-
+        successes = worker_pool.block_until_target_successes()
         ursulas_info = successes.values()
-        return ursulas_info
+        return list(ursulas_info)
 
     def _make_staker_reservoir(self,
                                quantity: int,
                                duration_periods: int,
-                               exclude_ursulas: List[str],
-                               include_ursulas: List[str]):
-        handpicked_ursulas = include_ursulas if include_ursulas else []
+                               exclude_ursulas: List[str] = None,
+                               include_ursulas: List[str] = None):
         if self.federated_only:
-            sample_size = quantity - len(handpicked_ursulas)
+            sample_size = quantity - (len(include_ursulas) if include_ursulas else 0)
             if not self.block_until_number_of_known_nodes_is(sample_size, learn_on_this_thread=True):
                 raise ValueError("Unable to learn about sufficient Ursulas")
             return make_federated_staker_reservoir(learner=self,
@@ -166,7 +175,7 @@ the Pipe for nucypher network operations
                                                    include_addresses=include_ursulas)
         else:
             return make_decentralized_staker_reservoir(staking_agent=self.staking_agent,
-                                                       periods=duration_periods,
+                                                       duration_periods=duration_periods,
                                                        exclude_addresses=exclude_ursulas,
                                                        include_addresses=include_ursulas)
 
@@ -227,11 +236,3 @@ the Pipe for nucypher network operations
             return response
 
         return controller
-
-
-class UrsulaInfo:
-    """Simple object that stores relevant Ursula information resulting from sampling."""
-    def __init__(self, checksum_address: str, ip_address: str, encrypting_key: UmbralPublicKey):
-        self.checksum_address = checksum_address
-        self.ip_address = ip_address
-        self.encrypting_key = encrypting_key

tests/acceptance/porter/test_decentralized_porter.py

@@ -0,0 +1,100 @@
+"""
+ This file is part of nucypher.
+
+ nucypher is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ nucypher is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import pytest
+from umbral.keys import UmbralPublicKey
+
+from nucypher.crypto.powers import DecryptingPower
+from nucypher.policy.collections import TreasureMap
+from tests.utils.middleware import MockRestMiddleware
+
+
+def test_get_ursulas(blockchain_porter, blockchain_ursulas):
+    # simple
+    quantity = 4
+    duration = 2  # irrelevant for federated
+    ursulas_info = blockchain_porter.get_ursulas(quantity=quantity, duration_periods=duration)
+    returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info}
+    assert len(returned_ursula_addresses) == quantity  # ensure no repeats
+
+    blockchain_ursulas_list = list(blockchain_ursulas)
+
+    # include specific ursulas
+    include_ursulas = [blockchain_ursulas_list[0].checksum_address, blockchain_ursulas_list[1].checksum_address]
+    ursulas_info = blockchain_porter.get_ursulas(quantity=quantity,
+                                                 duration_periods=duration,
+                                                 include_ursulas=include_ursulas)
+    returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info}
+    assert len(returned_ursula_addresses) == quantity
+    for address in include_ursulas:
+        assert address in returned_ursula_addresses
+
+    # exclude specific ursulas
+    number_to_exclude = len(blockchain_ursulas_list) - 4
+    exclude_ursulas = []
+    for i in range(number_to_exclude):
+        exclude_ursulas.append(blockchain_ursulas_list[i].checksum_address)
+    ursulas_info = blockchain_porter.get_ursulas(quantity=quantity,
+                                                 duration_periods=duration,
+                                                 exclude_ursulas=exclude_ursulas)
+    returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info}
+    assert len(returned_ursula_addresses) == quantity
+    for address in exclude_ursulas:
+        assert address not in returned_ursula_addresses
+
+    # include and exclude
+    include_ursulas = [blockchain_ursulas_list[0].checksum_address, blockchain_ursulas_list[1].checksum_address]
+    exclude_ursulas = [blockchain_ursulas_list[2].checksum_address, blockchain_ursulas_list[3].checksum_address]
+    ursulas_info = blockchain_porter.get_ursulas(quantity=quantity,
+                                                 duration_periods=duration,
+                                                 include_ursulas=include_ursulas,
+                                                 exclude_ursulas=exclude_ursulas)
+    returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info}
+    assert len(returned_ursula_addresses) == quantity
+    for address in include_ursulas:
+        assert address in returned_ursula_addresses
+    for address in exclude_ursulas:
+        assert address not in returned_ursula_addresses
+
+
+def test_publish_and_get_treasure_map(blockchain_porter,
+                                      blockchain_alice,
+                                      blockchain_bob,
+                                      idle_blockchain_policy):
+    # ensure that random treasure map cannot be obtained since not available
+    with pytest.raises(TreasureMap.NowhereToBeFound):
+        random_bob_encrypting_key = UmbralPublicKey.from_bytes(
+            bytes.fromhex("026d1f4ce5b2474e0dae499d6737a8d987ed3c9ab1a55e00f57ad2d8e81fe9e9ac"))
+        random_treasure_map_id = "93a9482bdf3b4f2e9df906a35144ca84"  # non-federated is 16 bytes
+        blockchain_porter.get_treasure_map(map_identifier=random_treasure_map_id,
+                                           bob_encrypting_key=random_bob_encrypting_key)
+
+    blockchain_bob_encrypting_key = blockchain_bob.public_keys(DecryptingPower)
+
+    # try publishing a new policy
+    network_middleware = MockRestMiddleware()
+    enacted_policy = idle_blockchain_policy.enact(network_middleware=network_middleware,
+                                                  publish_treasure_map=False)  # enact but don't publish
+    treasure_map = enacted_policy.treasure_map
+    blockchain_porter.publish_treasure_map(bytes(treasure_map), blockchain_bob_encrypting_key)
+
+    # try getting the recently published treasure map
+    map_id = blockchain_bob.construct_map_id(blockchain_alice.stamp,
+                                             enacted_policy.label)
+    retrieved_treasure_map = blockchain_porter.get_treasure_map(map_identifier=map_id,
+                                                                bob_encrypting_key=blockchain_bob_encrypting_key)
+    assert retrieved_treasure_map == treasure_map

tests/fixtures.py

@@ -63,6 +63,7 @@ from nucypher.crypto.powers import TransactingPower
 from nucypher.datastore import datastore
 from nucypher.network.nodes import TEACHER_NODES
 from nucypher.utilities.logging import GlobalLoggerSettings, Logger
+from nucypher.utilities.porter.porter import Porter
 from tests.constants import (
     BASE_TEMP_DIR,
     BASE_TEMP_PREFIX,
@@ -418,6 +419,35 @@ def lonely_ursula_maker(ursula_federated_test_config):
     _maker.clean()
 
 
+#
+# Porter
+#
+@pytest.fixture(scope="module")
+def federated_porter(federated_ursulas):
+    porter = Porter(domain=TEMPORARY_DOMAIN,
+                    abort_on_learning_error=True,
+                    start_learning_now=True,
+                    known_nodes=federated_ursulas,
+                    verify_node_bonding=False,
+                    federated_only=True,
+                    network_middleware=MockRestMiddleware())
+    yield porter
+    porter.stop_learning_loop()
+
+
+@pytest.fixture(scope="module")
+def blockchain_porter(blockchain_ursulas, testerchain, test_registry):
+    porter = Porter(domain=TEMPORARY_DOMAIN,
+                    abort_on_learning_error=True,
+                    start_learning_now=True,
+                    known_nodes=blockchain_ursulas,
+                    provider_uri=TEST_PROVIDER_URI,
+                    registry=test_registry,
+                    network_middleware=MockRestMiddleware())
+    yield porter
+    porter.stop_learning_loop()
+
+
 #
 # Blockchain
 #

tests/integration/porter/test_federated_porter.py

@@ -0,0 +1,107 @@
+"""
+ This file is part of nucypher.
+
+ nucypher is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ nucypher is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
+"""
+from base64 import b64decode
+
+import pytest
+from umbral.keys import UmbralPublicKey
+
+from nucypher.crypto.powers import DecryptingPower
+from nucypher.policy.collections import TreasureMap
+
+
+def test_get_ursulas(federated_porter, federated_ursulas):
+    # simple
+    quantity = 4
+    duration = 2  # irrelevant for federated
+    ursulas_info = federated_porter.get_ursulas(quantity=quantity, duration_periods=duration)
+    returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info}
+    assert len(returned_ursula_addresses) == quantity  # ensure no repeats
+
+    federated_ursulas_list = list(federated_ursulas)
+
+    # include specific ursulas
+    include_ursulas = [federated_ursulas_list[0].checksum_address, federated_ursulas_list[1].checksum_address]
+    ursulas_info = federated_porter.get_ursulas(quantity=quantity,
+                                                duration_periods=duration,
+                                                include_ursulas=include_ursulas)
+    returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info}
+    assert len(returned_ursula_addresses) == quantity
+    for address in include_ursulas:
+        assert address in returned_ursula_addresses
+
+    # exclude specific ursulas
+    number_to_exclude = len(federated_ursulas_list) - 4
+    exclude_ursulas = []
+    for i in range(number_to_exclude):
+        exclude_ursulas.append(federated_ursulas_list[i].checksum_address)
+    ursulas_info = federated_porter.get_ursulas(quantity=quantity,
+                                                duration_periods=duration,
+                                                exclude_ursulas=exclude_ursulas)
+    returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info}
+    assert len(returned_ursula_addresses) == quantity
+    for address in exclude_ursulas:
+        assert address not in returned_ursula_addresses
+
+    # include and exclude
+    include_ursulas = [federated_ursulas_list[0].checksum_address, federated_ursulas_list[1].checksum_address]
+    exclude_ursulas = [federated_ursulas_list[2].checksum_address, federated_ursulas_list[3].checksum_address]
+    ursulas_info = federated_porter.get_ursulas(quantity=quantity,
+                                                duration_periods=duration,
+                                                include_ursulas=include_ursulas,
+                                                exclude_ursulas=exclude_ursulas)
+    returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info}
+    assert len(returned_ursula_addresses) == quantity
+    for address in include_ursulas:
+        assert address in returned_ursula_addresses
+    for address in exclude_ursulas:
+        assert address not in returned_ursula_addresses
+
+
+def test_publish_and_get_treasure_map(federated_porter, federated_alice, federated_bob, enacted_federated_policy):
+    random_bob_encrypting_key = UmbralPublicKey.from_bytes(
+        bytes.fromhex("026d1f4ce5b2474e0dae499d6737a8d987ed3c9ab1a55e00f57ad2d8e81fe9e9ac"))
+    random_treasure_map_id = "f6ec73c93084ce91d5542a4ba6070071f5565112fe19b26ae9c960f9d658903a"  # federated is 32 bytes
+    random_treasure_map = b64decode("Qld7S8sbKFCv2B8KxfJo4oxiTOjZ4VPyqTK5K1xK6DND6TbLg2hvlGaMV69aiiC5QfadB82w/5q1"
+                                    "Sw+SNFHN2esWgAbs38QuUVUGCzDoWzQAAAGIAuhw12ZiPMNV8LaeWV8uUN+au2HGOjWilqtKsaP9f"
+                                    "mnLAzFiTUAu9/VCxOLOQE88BPoWk1H7OxRLDEhnBVYyflpifKbOYItwLLTtWYVFRY90LtNSAzS8d3v"
+                                    "NH4c3SHSZwYsCKY+5LvJ68GD0CqhydSxCcGckh0unttHrYGSOQsURUI4AAAEBsSMlukjA1WyYA+Fouq"
+                                    "kuRtk8bVHcYLqRUkK2n6dShEUGMuY1SzcAbBINvJYmQp+hhzK5m47AzCl463emXepYZQC/evytktG7y"
+                                    "Xxd3k8Ak+Qr7T4+G2VgJl4YrafTpIT6wowd+8u/SMSrrf/M41OhtLeBC4uDKjO3rYBQfVLTpEAgiX/9"
+                                    "jxB80RtNMeCwgcieviAR5tlw2IlxVTEhxXbFeopcOZmfEuhVWqgBUfIakqsNCXkkubV0XS2l5G1vtTM8"
+                                    "oNML0rP8PyKd4+0M5N6P/EQqFkHH93LCDD0IQBq9usm3MoJp0eT8N3m5gprI05drDh2xe/W6qnQfw3YXn"
+                                    "jdvf2A=")
+
+    # ensure that random treasure map cannot be obtained since not available
+    with pytest.raises(TreasureMap.NowhereToBeFound):
+        federated_porter.get_treasure_map(map_identifier=random_treasure_map_id,
+                                          bob_encrypting_key=random_bob_encrypting_key)
+
+    # publish the random treasure map
+    federated_porter.publish_treasure_map(treasure_map_bytes=random_treasure_map,
+                                          bob_encrypting_key=random_bob_encrypting_key)
+
+    # try getting the random treasure map now
+    treasure_map = federated_porter.get_treasure_map(map_identifier=random_treasure_map_id,
+                                                     bob_encrypting_key=random_bob_encrypting_key)
+    assert treasure_map.public_id() == random_treasure_map_id
+
+    # try getting an already existing policy
+    map_id = federated_bob.construct_map_id(federated_alice.stamp,
+                                            enacted_federated_policy.label)
+    treasure_map = federated_porter.get_treasure_map(map_identifier=map_id,
+                                                     bob_encrypting_key=federated_bob.public_keys(DecryptingPower))
+    assert treasure_map == enacted_federated_policy.treasure_map

tests/integration/porter/test_porter_specifications.py

@@ -192,7 +192,7 @@ def test_alice_revoke():
     pass  # TODO
 
 
-def test_bob_get_treasure_map(enacted_federated_policy, federated_bob):
+def test_bob_get_treasure_map(enacted_federated_policy, federated_alice, federated_bob):
     #
     # Input i.e. load
     #
@@ -201,7 +201,7 @@ def test_bob_get_treasure_map(enacted_federated_policy, federated_bob):
     with pytest.raises(InvalidInputData):
         BobGetTreasureMap().load({})
 
-    treasure_map_id = enacted_federated_policy.treasure_map.public_id()
+    treasure_map_id = federated_bob.construct_map_id(federated_alice.stamp, enacted_federated_policy.label)
     bob_encrypting_key = federated_bob.public_keys(DecryptingPower)
     bob_encrypting_key_hex = bytes(bob_encrypting_key).hex()
 
@@ -213,6 +213,11 @@ def test_bob_get_treasure_map(enacted_federated_policy, federated_bob):
     # required args
     BobGetTreasureMap().load(required_data)
 
+    # random 16-byte length map id
+    updated_data = dict(required_data)
+    updated_data['treasure_map_id'] = "93a9482bdf3b4f2e9df906a35144ca93"
+    BobGetTreasureMap().load(updated_data)
+
     # missing required args
     updated_data = {k: v for k, v in required_data.items() if k != 'treasure_map_id'}
     with pytest.raises(InvalidInputData):