From dbeec7f1ae607e617fc151511b83eba805827204 Mon Sep 17 00:00:00 2001 From: derekpierre Date: Fri, 4 Jun 2021 13:31:48 -0400 Subject: [PATCH] Add integration and acceptance tests for porter functionality. Remove unused 'publish_treasure_map' parameter from _enact() function. Loosen validation on treasure map id which can be 32 bytes (federated) or 16 bytes (non-federated). Probably only care about non-federated but for ease of testing, federated really helps. --- nucypher/cli/commands/porter.py | 3 +- nucypher/policy/policies.py | 2 +- nucypher/policy/reservoir.py | 12 +- .../specifications/fields/treasuremapid.py | 5 +- nucypher/utilities/porter/porter.py | 61 +++++----- .../porter/test_decentralized_porter.py | 100 ++++++++++++++++ tests/fixtures.py | 30 +++++ .../porter/test_federated_porter.py | 107 ++++++++++++++++++ .../porter/test_porter_specifications.py | 9 +- 9 files changed, 289 insertions(+), 40 deletions(-) create mode 100644 tests/acceptance/porter/test_decentralized_porter.py create mode 100644 tests/integration/porter/test_federated_porter.py diff --git a/nucypher/cli/commands/porter.py b/nucypher/cli/commands/porter.py index c7b764b6c..8a930d9e2 100644 --- a/nucypher/cli/commands/porter.py +++ b/nucypher/cli/commands/porter.py @@ -112,7 +112,8 @@ def run(general_config, network, provider_uri, federated_only, teacher_uri, http BlockchainInterfaceFactory.initialize_interface(provider_uri=provider_uri) PORTER = Porter(domain=network, - start_learning_now=eager) + start_learning_now=eager, + provider_uri=provider_uri) # RPC if general_config.json_ipc: diff --git a/nucypher/policy/policies.py b/nucypher/policy/policies.py index eedf91506..4a7e1fc32 100644 --- a/nucypher/policy/policies.py +++ b/nucypher/policy/policies.py @@ -515,7 +515,7 @@ class BlockchainPolicy(Policy): def _make_reservoir(self, handpicked_addresses): staker_reservoir = make_decentralized_staker_reservoir(staking_agent=self.alice.staking_agent, - periods=self.payment_periods, + duration_periods=self.payment_periods, include_addresses=handpicked_addresses) return staker_reservoir diff --git a/nucypher/policy/reservoir.py b/nucypher/policy/reservoir.py index 64942e0f0..ed2b0956e 100644 --- a/nucypher/policy/reservoir.py +++ b/nucypher/policy/reservoir.py @@ -21,7 +21,9 @@ from eth_typing import ChecksumAddress from nucypher.blockchain.eth.agents import StakersReservoir, StakingEscrowAgent -def make_federated_staker_reservoir(learner: 'Learner', exclude_addresses=None, include_addresses=None): +def make_federated_staker_reservoir(learner: 'Learner', + exclude_addresses: List[str] = None, + include_addresses: List[str] = None): """ Get a sampler object containing the federated stakers. """ @@ -45,9 +47,9 @@ def make_federated_staker_reservoir(learner: 'Learner', exclude_addresses=None, def make_decentralized_staker_reservoir(staking_agent: StakingEscrowAgent, - periods, - exclude_addresses=None, - include_addresses=None): + duration_periods: int, + exclude_addresses: List[str] = None, + include_addresses: List[str] = None): """ Get a sampler object containing the currently registered stakers. """ @@ -60,7 +62,7 @@ def make_decentralized_staker_reservoir(staking_agent: StakingEscrowAgent, without_set.update(exclude_addresses) without_set.update(include_addresses) try: - reservoir = staking_agent.get_stakers_reservoir(duration=periods, + reservoir = staking_agent.get_stakers_reservoir(duration=duration_periods, without=without_set) except StakingEscrowAgent.NotEnoughStakers: # TODO: do that in `get_stakers_reservoir()`? diff --git a/nucypher/utilities/porter/control/specifications/fields/treasuremapid.py b/nucypher/utilities/porter/control/specifications/fields/treasuremapid.py index 40c3c4c63..1f4de5354 100644 --- a/nucypher/utilities/porter/control/specifications/fields/treasuremapid.py +++ b/nucypher/utilities/porter/control/specifications/fields/treasuremapid.py @@ -19,11 +19,14 @@ from marshmallow import fields from nucypher.control.specifications.exceptions import InvalidInputData from nucypher.control.specifications.fields.base import BaseField +from nucypher.crypto.constants import HRAC_LENGTH from nucypher.policy.collections import TreasureMap class TreasureMapID(BaseField, fields.String): def _validate(self, value): - if len(bytes.fromhex(value)) != TreasureMap.ID_LENGTH: + treasure_map_id = bytes.fromhex(value) + # FIXME federated has map id length 32 bytes but decentralized has length 16 bytes ... huh? + if len(treasure_map_id) != TreasureMap.ID_LENGTH and len(treasure_map_id) != HRAC_LENGTH: raise InvalidInputData(f"Could not convert input for {self.name} to a valid TreasureMap ID: invalid length") diff --git a/nucypher/utilities/porter/porter.py b/nucypher/utilities/porter/porter.py index 5a79d5f5a..fedeed654 100644 --- a/nucypher/utilities/porter/porter.py +++ b/nucypher/utilities/porter/porter.py @@ -21,6 +21,7 @@ from flask import request, Response from umbral.keys import UmbralPublicKey from nucypher.blockchain.eth.agents import ContractAgency, StakingEscrowAgent +from nucypher.blockchain.eth.interfaces import BlockchainInterfaceFactory from nucypher.blockchain.eth.registry import BaseContractRegistry, InMemoryContractRegistry from nucypher.characters import utils from nucypher.characters.lawful import Ursula @@ -64,16 +65,30 @@ the Pipe for nucypher network operations _interface_class = PorterInterface + class UrsulaInfo: + """Simple object that stores relevant Ursula information resulting from sampling.""" + + def __init__(self, checksum_address: str, ip_address: str, encrypting_key: UmbralPublicKey): + self.checksum_address = checksum_address + self.ip_address = ip_address + self.encrypting_key = encrypting_key + def __init__(self, domain: str = None, registry: BaseContractRegistry = None, controller: bool = True, federated_only: bool = False, node_class: object = Ursula, + provider_uri: str = None, *args, **kwargs): self.federated_only = federated_only if not self.federated_only: + if not provider_uri: + if not provider_uri: + raise ValueError('Provider URI is required for decentralized Porter.') + + BlockchainInterfaceFactory.get_interface(provider_uri=provider_uri) self.registry = registry or InMemoryContractRegistry.from_latest_publication(network=domain) self.staking_agent = ContractAgency.get_agent(StakingEscrowAgent, registry=self.registry) else: @@ -110,7 +125,11 @@ the Pipe for nucypher network operations treasure_map_publisher.block_until_success_is_reasonably_likely() return - def get_ursulas(self, quantity: int, duration_periods: int, exclude_ursulas: List[str], include_ursulas: List[str]): + def get_ursulas(self, + quantity: int, + duration_periods: int, + exclude_ursulas: List[str] = None, + include_ursulas: List[str] = None) -> List[UrsulaInfo]: reservoir = self._make_staker_reservoir(quantity, duration_periods, exclude_ursulas, include_ursulas) value_factory = PrefetchStrategy(reservoir, quantity) @@ -118,16 +137,16 @@ the Pipe for nucypher network operations if ursula_checksum not in self.known_nodes: raise ValueError(f"{ursula_checksum} is not known") - # check connectivity ursula = self.known_nodes[ursula_checksum] - # don't care about the result only that it worked without raising an exception + # check connectivity - don't care about the result only that it worked without raising an exception # TODO is this the best way to check connectivity? _ = self.network_middleware.get_certificate(host=ursula.rest_interface.host, port=ursula.rest_interface.port) - return UrsulaInfo(checksum_address=ursula_checksum, - ip_address=f"https://{ursula.rest_interface.host}:{ursula.rest_interface.port}", - encrypting_key=ursula.public_keys(DecryptingPower)) + + return Porter.UrsulaInfo(checksum_address=ursula_checksum, + ip_address=f"https://{ursula.rest_interface.host}:{ursula.rest_interface.port}", + encrypting_key=ursula.public_keys(DecryptingPower)) self.block_until_number_of_known_nodes_is(quantity, learn_on_this_thread=True, eager=True) @@ -138,27 +157,17 @@ the Pipe for nucypher network operations stagger_timeout=1, threadpool_size=quantity) worker_pool.start() - try: - successes = worker_pool.block_until_target_successes() - except (WorkerPool.OutOfValues, WorkerPool.TimedOut): - # It's possible to raise some other exceptions here, - # but we will use the logic below. - successes = worker_pool.get_successes() - finally: - worker_pool.cancel() - worker_pool.join() - + successes = worker_pool.block_until_target_successes() ursulas_info = successes.values() - return ursulas_info + return list(ursulas_info) def _make_staker_reservoir(self, quantity: int, duration_periods: int, - exclude_ursulas: List[str], - include_ursulas: List[str]): - handpicked_ursulas = include_ursulas if include_ursulas else [] + exclude_ursulas: List[str] = None, + include_ursulas: List[str] = None): if self.federated_only: - sample_size = quantity - len(handpicked_ursulas) + sample_size = quantity - (len(include_ursulas) if include_ursulas else 0) if not self.block_until_number_of_known_nodes_is(sample_size, learn_on_this_thread=True): raise ValueError("Unable to learn about sufficient Ursulas") return make_federated_staker_reservoir(learner=self, @@ -166,7 +175,7 @@ the Pipe for nucypher network operations include_addresses=include_ursulas) else: return make_decentralized_staker_reservoir(staking_agent=self.staking_agent, - periods=duration_periods, + duration_periods=duration_periods, exclude_addresses=exclude_ursulas, include_addresses=include_ursulas) @@ -227,11 +236,3 @@ the Pipe for nucypher network operations return response return controller - - -class UrsulaInfo: - """Simple object that stores relevant Ursula information resulting from sampling.""" - def __init__(self, checksum_address: str, ip_address: str, encrypting_key: UmbralPublicKey): - self.checksum_address = checksum_address - self.ip_address = ip_address - self.encrypting_key = encrypting_key diff --git a/tests/acceptance/porter/test_decentralized_porter.py b/tests/acceptance/porter/test_decentralized_porter.py new file mode 100644 index 000000000..de45547d7 --- /dev/null +++ b/tests/acceptance/porter/test_decentralized_porter.py @@ -0,0 +1,100 @@ +""" + This file is part of nucypher. + + nucypher is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + nucypher is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with nucypher. If not, see . +""" + +import pytest +from umbral.keys import UmbralPublicKey + +from nucypher.crypto.powers import DecryptingPower +from nucypher.policy.collections import TreasureMap +from tests.utils.middleware import MockRestMiddleware + + +def test_get_ursulas(blockchain_porter, blockchain_ursulas): + # simple + quantity = 4 + duration = 2 # irrelevant for federated + ursulas_info = blockchain_porter.get_ursulas(quantity=quantity, duration_periods=duration) + returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info} + assert len(returned_ursula_addresses) == quantity # ensure no repeats + + blockchain_ursulas_list = list(blockchain_ursulas) + + # include specific ursulas + include_ursulas = [blockchain_ursulas_list[0].checksum_address, blockchain_ursulas_list[1].checksum_address] + ursulas_info = blockchain_porter.get_ursulas(quantity=quantity, + duration_periods=duration, + include_ursulas=include_ursulas) + returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info} + assert len(returned_ursula_addresses) == quantity + for address in include_ursulas: + assert address in returned_ursula_addresses + + # exclude specific ursulas + number_to_exclude = len(blockchain_ursulas_list) - 4 + exclude_ursulas = [] + for i in range(number_to_exclude): + exclude_ursulas.append(blockchain_ursulas_list[i].checksum_address) + ursulas_info = blockchain_porter.get_ursulas(quantity=quantity, + duration_periods=duration, + exclude_ursulas=exclude_ursulas) + returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info} + assert len(returned_ursula_addresses) == quantity + for address in exclude_ursulas: + assert address not in returned_ursula_addresses + + # include and exclude + include_ursulas = [blockchain_ursulas_list[0].checksum_address, blockchain_ursulas_list[1].checksum_address] + exclude_ursulas = [blockchain_ursulas_list[2].checksum_address, blockchain_ursulas_list[3].checksum_address] + ursulas_info = blockchain_porter.get_ursulas(quantity=quantity, + duration_periods=duration, + include_ursulas=include_ursulas, + exclude_ursulas=exclude_ursulas) + returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info} + assert len(returned_ursula_addresses) == quantity + for address in include_ursulas: + assert address in returned_ursula_addresses + for address in exclude_ursulas: + assert address not in returned_ursula_addresses + + +def test_publish_and_get_treasure_map(blockchain_porter, + blockchain_alice, + blockchain_bob, + idle_blockchain_policy): + # ensure that random treasure map cannot be obtained since not available + with pytest.raises(TreasureMap.NowhereToBeFound): + random_bob_encrypting_key = UmbralPublicKey.from_bytes( + bytes.fromhex("026d1f4ce5b2474e0dae499d6737a8d987ed3c9ab1a55e00f57ad2d8e81fe9e9ac")) + random_treasure_map_id = "93a9482bdf3b4f2e9df906a35144ca84" # non-federated is 16 bytes + blockchain_porter.get_treasure_map(map_identifier=random_treasure_map_id, + bob_encrypting_key=random_bob_encrypting_key) + + blockchain_bob_encrypting_key = blockchain_bob.public_keys(DecryptingPower) + + # try publishing a new policy + network_middleware = MockRestMiddleware() + enacted_policy = idle_blockchain_policy.enact(network_middleware=network_middleware, + publish_treasure_map=False) # enact but don't publish + treasure_map = enacted_policy.treasure_map + blockchain_porter.publish_treasure_map(bytes(treasure_map), blockchain_bob_encrypting_key) + + # try getting the recently published treasure map + map_id = blockchain_bob.construct_map_id(blockchain_alice.stamp, + enacted_policy.label) + retrieved_treasure_map = blockchain_porter.get_treasure_map(map_identifier=map_id, + bob_encrypting_key=blockchain_bob_encrypting_key) + assert retrieved_treasure_map == treasure_map diff --git a/tests/fixtures.py b/tests/fixtures.py index 0f22537f1..d920220fb 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -63,6 +63,7 @@ from nucypher.crypto.powers import TransactingPower from nucypher.datastore import datastore from nucypher.network.nodes import TEACHER_NODES from nucypher.utilities.logging import GlobalLoggerSettings, Logger +from nucypher.utilities.porter.porter import Porter from tests.constants import ( BASE_TEMP_DIR, BASE_TEMP_PREFIX, @@ -418,6 +419,35 @@ def lonely_ursula_maker(ursula_federated_test_config): _maker.clean() +# +# Porter +# +@pytest.fixture(scope="module") +def federated_porter(federated_ursulas): + porter = Porter(domain=TEMPORARY_DOMAIN, + abort_on_learning_error=True, + start_learning_now=True, + known_nodes=federated_ursulas, + verify_node_bonding=False, + federated_only=True, + network_middleware=MockRestMiddleware()) + yield porter + porter.stop_learning_loop() + + +@pytest.fixture(scope="module") +def blockchain_porter(blockchain_ursulas, testerchain, test_registry): + porter = Porter(domain=TEMPORARY_DOMAIN, + abort_on_learning_error=True, + start_learning_now=True, + known_nodes=blockchain_ursulas, + provider_uri=TEST_PROVIDER_URI, + registry=test_registry, + network_middleware=MockRestMiddleware()) + yield porter + porter.stop_learning_loop() + + # # Blockchain # diff --git a/tests/integration/porter/test_federated_porter.py b/tests/integration/porter/test_federated_porter.py new file mode 100644 index 000000000..d670f2ba4 --- /dev/null +++ b/tests/integration/porter/test_federated_porter.py @@ -0,0 +1,107 @@ +""" + This file is part of nucypher. + + nucypher is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + nucypher is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with nucypher. If not, see . +""" +from base64 import b64decode + +import pytest +from umbral.keys import UmbralPublicKey + +from nucypher.crypto.powers import DecryptingPower +from nucypher.policy.collections import TreasureMap + + +def test_get_ursulas(federated_porter, federated_ursulas): + # simple + quantity = 4 + duration = 2 # irrelevant for federated + ursulas_info = federated_porter.get_ursulas(quantity=quantity, duration_periods=duration) + returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info} + assert len(returned_ursula_addresses) == quantity # ensure no repeats + + federated_ursulas_list = list(federated_ursulas) + + # include specific ursulas + include_ursulas = [federated_ursulas_list[0].checksum_address, federated_ursulas_list[1].checksum_address] + ursulas_info = federated_porter.get_ursulas(quantity=quantity, + duration_periods=duration, + include_ursulas=include_ursulas) + returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info} + assert len(returned_ursula_addresses) == quantity + for address in include_ursulas: + assert address in returned_ursula_addresses + + # exclude specific ursulas + number_to_exclude = len(federated_ursulas_list) - 4 + exclude_ursulas = [] + for i in range(number_to_exclude): + exclude_ursulas.append(federated_ursulas_list[i].checksum_address) + ursulas_info = federated_porter.get_ursulas(quantity=quantity, + duration_periods=duration, + exclude_ursulas=exclude_ursulas) + returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info} + assert len(returned_ursula_addresses) == quantity + for address in exclude_ursulas: + assert address not in returned_ursula_addresses + + # include and exclude + include_ursulas = [federated_ursulas_list[0].checksum_address, federated_ursulas_list[1].checksum_address] + exclude_ursulas = [federated_ursulas_list[2].checksum_address, federated_ursulas_list[3].checksum_address] + ursulas_info = federated_porter.get_ursulas(quantity=quantity, + duration_periods=duration, + include_ursulas=include_ursulas, + exclude_ursulas=exclude_ursulas) + returned_ursula_addresses = {ursula_info.checksum_address for ursula_info in ursulas_info} + assert len(returned_ursula_addresses) == quantity + for address in include_ursulas: + assert address in returned_ursula_addresses + for address in exclude_ursulas: + assert address not in returned_ursula_addresses + + +def test_publish_and_get_treasure_map(federated_porter, federated_alice, federated_bob, enacted_federated_policy): + random_bob_encrypting_key = UmbralPublicKey.from_bytes( + bytes.fromhex("026d1f4ce5b2474e0dae499d6737a8d987ed3c9ab1a55e00f57ad2d8e81fe9e9ac")) + random_treasure_map_id = "f6ec73c93084ce91d5542a4ba6070071f5565112fe19b26ae9c960f9d658903a" # federated is 32 bytes + random_treasure_map = b64decode("Qld7S8sbKFCv2B8KxfJo4oxiTOjZ4VPyqTK5K1xK6DND6TbLg2hvlGaMV69aiiC5QfadB82w/5q1" + "Sw+SNFHN2esWgAbs38QuUVUGCzDoWzQAAAGIAuhw12ZiPMNV8LaeWV8uUN+au2HGOjWilqtKsaP9f" + "mnLAzFiTUAu9/VCxOLOQE88BPoWk1H7OxRLDEhnBVYyflpifKbOYItwLLTtWYVFRY90LtNSAzS8d3v" + "NH4c3SHSZwYsCKY+5LvJ68GD0CqhydSxCcGckh0unttHrYGSOQsURUI4AAAEBsSMlukjA1WyYA+Fouq" + "kuRtk8bVHcYLqRUkK2n6dShEUGMuY1SzcAbBINvJYmQp+hhzK5m47AzCl463emXepYZQC/evytktG7y" + "Xxd3k8Ak+Qr7T4+G2VgJl4YrafTpIT6wowd+8u/SMSrrf/M41OhtLeBC4uDKjO3rYBQfVLTpEAgiX/9" + "jxB80RtNMeCwgcieviAR5tlw2IlxVTEhxXbFeopcOZmfEuhVWqgBUfIakqsNCXkkubV0XS2l5G1vtTM8" + "oNML0rP8PyKd4+0M5N6P/EQqFkHH93LCDD0IQBq9usm3MoJp0eT8N3m5gprI05drDh2xe/W6qnQfw3YXn" + "jdvf2A=") + + # ensure that random treasure map cannot be obtained since not available + with pytest.raises(TreasureMap.NowhereToBeFound): + federated_porter.get_treasure_map(map_identifier=random_treasure_map_id, + bob_encrypting_key=random_bob_encrypting_key) + + # publish the random treasure map + federated_porter.publish_treasure_map(treasure_map_bytes=random_treasure_map, + bob_encrypting_key=random_bob_encrypting_key) + + # try getting the random treasure map now + treasure_map = federated_porter.get_treasure_map(map_identifier=random_treasure_map_id, + bob_encrypting_key=random_bob_encrypting_key) + assert treasure_map.public_id() == random_treasure_map_id + + # try getting an already existing policy + map_id = federated_bob.construct_map_id(federated_alice.stamp, + enacted_federated_policy.label) + treasure_map = federated_porter.get_treasure_map(map_identifier=map_id, + bob_encrypting_key=federated_bob.public_keys(DecryptingPower)) + assert treasure_map == enacted_federated_policy.treasure_map diff --git a/tests/integration/porter/test_porter_specifications.py b/tests/integration/porter/test_porter_specifications.py index 97416f092..24900608c 100644 --- a/tests/integration/porter/test_porter_specifications.py +++ b/tests/integration/porter/test_porter_specifications.py @@ -192,7 +192,7 @@ def test_alice_revoke(): pass # TODO -def test_bob_get_treasure_map(enacted_federated_policy, federated_bob): +def test_bob_get_treasure_map(enacted_federated_policy, federated_alice, federated_bob): # # Input i.e. load # @@ -201,7 +201,7 @@ def test_bob_get_treasure_map(enacted_federated_policy, federated_bob): with pytest.raises(InvalidInputData): BobGetTreasureMap().load({}) - treasure_map_id = enacted_federated_policy.treasure_map.public_id() + treasure_map_id = federated_bob.construct_map_id(federated_alice.stamp, enacted_federated_policy.label) bob_encrypting_key = federated_bob.public_keys(DecryptingPower) bob_encrypting_key_hex = bytes(bob_encrypting_key).hex() @@ -213,6 +213,11 @@ def test_bob_get_treasure_map(enacted_federated_policy, federated_bob): # required args BobGetTreasureMap().load(required_data) + # random 16-byte length map id + updated_data = dict(required_data) + updated_data['treasure_map_id'] = "93a9482bdf3b4f2e9df906a35144ca93" + BobGetTreasureMap().load(updated_data) + # missing required args updated_data = {k: v for k, v in required_data.items() if k != 'treasure_map_id'} with pytest.raises(InvalidInputData):