From febc6093c1496e2daa88bf23c9db577dba2a4382 Mon Sep 17 00:00:00 2001 From: Kieran Prasch Date: Wed, 26 Oct 2022 15:37:45 +0100 Subject: [PATCH] Removes lmdb datastore usage --- examples/run_demo_ursula_fleet.py | 7 +--- nucypher/characters/lawful.py | 21 ++--------- nucypher/cli/actions/confirm.py | 7 +--- nucypher/cli/literature.py | 1 - nucypher/cli/options.py | 1 - nucypher/cli/painting/nodes.py | 5 --- nucypher/config/characters.py | 7 ---- nucypher/network/server.py | 35 +++++-------------- nucypher/utilities/prometheus/collector.py | 6 ---- .../blockchain/actors/test_operator.py | 3 -- .../characters/test_ursula_web_status.py | 5 +-- tests/conftest.py | 10 +----- tests/fixtures.py | 6 ---- .../config/test_character_configuration.py | 2 -- .../config/test_keystore_integration.py | 2 -- .../learning/test_discovery_phases.py | 13 +------ tests/mock/performance_mocks.py | 16 ++++----- tests/utils/ursula.py | 3 -- 18 files changed, 23 insertions(+), 127 deletions(-) diff --git a/examples/run_demo_ursula_fleet.py b/examples/run_demo_ursula_fleet.py index b7d383199..e2f3d64dc 100644 --- a/examples/run_demo_ursula_fleet.py +++ b/examples/run_demo_ursula_fleet.py @@ -65,12 +65,7 @@ def spin_up_federated_ursulas(quantity: int = FLEET_POPULATION): deployer.start() print(f"{u}: {deployer._listening_message()}") - try: - reactor.run() - finally: - for u in ursulas: - with suppress(FileNotFoundError): - shutil.rmtree(u.datastore.db_path) + reactor.run() if __name__ == "__main__": diff --git a/nucypher/characters/lawful.py b/nucypher/characters/lawful.py index 83a4ab8c0..1a8c0d306 100644 --- a/nucypher/characters/lawful.py +++ b/nucypher/characters/lawful.py @@ -683,9 +683,7 @@ class Ursula(Teacher, Character, Operator): self.payment_method = payment_method # Server - self.rest_server = self._make_local_server(host=rest_host, - port=rest_port, - db_filepath=db_filepath) + self.rest_server = self._make_local_server(host=rest_host, port=rest_port) # Self-signed TLS certificate of self for Teacher.__init__ certificate_filepath = self._crypto_power.power_ups(TLSHostingPower).keypair.certificate_filepath @@ -727,15 +725,11 @@ class Ursula(Teacher, Character, Operator): self._crypto_power.consume_power_up(tls_hosting_power) # Consume! return tls_hosting_power - def _make_local_server(self, host, port, db_filepath) -> ProxyRESTServer: - rest_app, datastore = make_rest_app( - this_node=self, - db_filepath=db_filepath, - ) + def _make_local_server(self, host, port) -> ProxyRESTServer: + rest_app = make_rest_app(this_node=self) rest_server = ProxyRESTServer(rest_host=host, rest_port=port, rest_app=rest_app, - datastore=datastore, hosting_power=self.__get_hosting_power(host=host)) return rest_server @@ -899,9 +893,6 @@ class Ursula(Teacher, Character, Operator): **Warning:** invalidates the Ursula. """ - - # `rest_server` holds references to the datastore (directly and via `rest_app`). - # An open datastore hogs up file descriptors. self.rest_server = INVALIDATED def rest_information(self): @@ -1083,12 +1074,6 @@ class Ursula(Teacher, Character, Operator): # # Properties # - @property - def datastore(self): - try: - return self.rest_server.datastore - except AttributeError: - raise AttributeError("No rest server attached") @property def rest_url(self): diff --git a/nucypher/cli/actions/confirm.py b/nucypher/cli/actions/confirm.py index e96457945..222435c9e 100644 --- a/nucypher/cli/actions/confirm.py +++ b/nucypher/cli/actions/confirm.py @@ -57,16 +57,11 @@ def confirm_deployment(emitter: StdoutEmitter, deployer_interface: BlockchainDep def confirm_destroy_configuration(config: CharacterConfiguration) -> bool: """Interactively confirm destruction of nucypher configuration files""" # TODO: This is a workaround for ursula - needs follow up - try: - database = config.db_filepath - except AttributeError: - database = "No database found" confirmation = CHARACTER_DESTRUCTION.format(name=config.NAME, root=config.config_root, keystore=config.keystore_dir, nodestore=config.node_storage.source, - config=config.filepath, - database=database) + config=config.filepath) click.confirm(confirmation, abort=True) return True diff --git a/nucypher/cli/literature.py b/nucypher/cli/literature.py index a18c45480..18891d6db 100644 --- a/nucypher/cli/literature.py +++ b/nucypher/cli/literature.py @@ -126,7 +126,6 @@ Delete all {name} character files including: - Private and Public Keys ({keystore}) - Known Nodes ({nodestore}) - Node Configuration File ({config}) - - Database ({database}) Are you sure?""" diff --git a/nucypher/cli/options.py b/nucypher/cli/options.py index f4fd0ce9a..90f4adf83 100644 --- a/nucypher/cli/options.py +++ b/nucypher/cli/options.py @@ -41,7 +41,6 @@ from nucypher.utilities.logging import Logger option_config_file = click.option('--config-file', help="Path to configuration file", type=EXISTING_READABLE_FILE) option_config_root = click.option('--config-root', help="Custom configuration directory", type=click.Path(path_type=Path)) option_dev = click.option('--dev', '-d', help="Enable development mode", is_flag=True) -option_db_filepath = click.option('--db-filepath', help="The database filepath to connect to", type=click.Path(path_type=Path)) option_dry_run = click.option('--dry-run', '-x', help="Execute normally without actually starting the node", is_flag=True) option_etherscan = click.option('--etherscan/--no-etherscan', help="Enable/disable viewing TX in Etherscan") option_event_name = click.option('--event-name', help="Specify an event by name", type=click.STRING) diff --git a/nucypher/cli/painting/nodes.py b/nucypher/cli/painting/nodes.py index 18894895e..4e71ac170 100644 --- a/nucypher/cli/painting/nodes.py +++ b/nucypher/cli/painting/nodes.py @@ -17,7 +17,6 @@ along with nucypher. If not, see . import maya from nucypher.config.constants import SEEDNODES -from nucypher.datastore.queries import get_reencryption_requests def build_fleet_state_status(ursula) -> str: @@ -41,9 +40,6 @@ def paint_node_status(emitter, ursula, start_time): # Build FleetState status line fleet_state = build_fleet_state_status(ursula=ursula) - reenc_requests = get_reencryption_requests(ursula.datastore) - num_reenc_requests = len(reenc_requests) - stats = ['⇀URSULA {}↽'.format(ursula.nickname.icon), '{}'.format(ursula), 'Uptime .............. {}'.format(maya.now() - start_time), @@ -55,7 +51,6 @@ def paint_node_status(emitter, ursula, start_time): 'Rest Interface ...... {}'.format(ursula.rest_url()), 'Node Storage Type ... {}'.format(ursula.node_storage._name.capitalize()), 'Known Nodes ......... {}'.format(len(ursula.known_nodes)), - 'Reencryption Requests {}'.format(num_reenc_requests), teacher] if not ursula.federated_only: diff --git a/nucypher/config/characters.py b/nucypher/config/characters.py index bdd9820ee..99b8fa437 100644 --- a/nucypher/config/characters.py +++ b/nucypher/config/characters.py @@ -130,13 +130,6 @@ class UrsulaConfiguration(CharacterConfiguration): merged_parameters = self.generate_parameters(**overrides) ursula = self.CHARACTER_CLASS(**merged_parameters) - - if self.dev_mode: - class MockDatastoreThreadPool(object): - def callInThread(self, f, *args, **kwargs): - return f(*args, **kwargs) - ursula.datastore_threadpool = MockDatastoreThreadPool() - return ursula def destroy(self) -> None: diff --git a/nucypher/network/server.py b/nucypher/network/server.py index 0d3a25415..2e464ebf1 100644 --- a/nucypher/network/server.py +++ b/nucypher/network/server.py @@ -37,8 +37,6 @@ from nucypher_core import ( from nucypher.config.constants import MAX_UPLOAD_CONTENT_LENGTH from nucypher.crypto.keypairs import DecryptingKeypair from nucypher.crypto.signing import InvalidSignature -from nucypher.datastore.datastore import Datastore -from nucypher.datastore.models import ReencryptionRequest as ReencryptionRequestModel from nucypher.network.exceptions import NodeSeemsToBeDown from nucypher.network.nodes import NodeSprout from nucypher.network.protocols import InterfaceInfo @@ -65,14 +63,12 @@ class ProxyRESTServer: rest_host: str, rest_port: int, hosting_power=None, - rest_app=None, - datastore=None, + rest_app=None ) -> None: self.rest_interface = InterfaceInfo(host=rest_host, port=rest_port) if rest_app: # if is me self.rest_app = rest_app - self.datastore = datastore else: self.rest_app = constants.PUBLIC_ONLY @@ -83,30 +79,20 @@ class ProxyRESTServer: def make_rest_app( - db_filepath: Path, this_node, log: Logger = Logger("http-application-layer") - ) -> Tuple[Flask, Datastore]: - """ - Creates a REST application and an associated ``Datastore`` object. - Note that the REST app **does not** hold a reference to the datastore; - it is your responsibility to ensure it lives for as long as the app does. - """ + ) -> Flask: + """Creates a REST application.""" # A trampoline function for the real REST app, - # to ensure that a reference to the node and the datastore object is not held by the app closure. + # to ensure that a reference to the node object is not held by the app closure. # One would think that it's enough to only remove a reference to the node, - # but `rest_app` somehow holds a reference to itself, Uroboros-like, - # and will hold the datastore reference if it is created there. - - log.info("Starting datastore {}".format(db_filepath)) - datastore = Datastore(db_filepath) - rest_app = _make_rest_app(weakref.proxy(datastore), weakref.proxy(this_node), log) - - return rest_app, datastore + # but `rest_app` somehow holds a reference to itself, Uroboros-like... + rest_app = _make_rest_app(weakref.proxy(this_node), log) + return rest_app -def _make_rest_app(datastore: Datastore, this_node, log: Logger) -> Flask: +def _make_rest_app(this_node, log: Logger) -> Flask: # TODO: Avoid circular imports :-( from nucypher.characters.lawful import Alice, Bob, Ursula @@ -292,11 +278,6 @@ def _make_rest_app(datastore: Datastore, this_node, log: Logger) -> Flask: # TODO: return a sensible response if it fails (currently results in 500) response = this_node._reencrypt(kfrag=verified_kfrag, capsules=capsules_to_process) - # Now, Ursula saves evidence of this workorder to her database... - # Note: we give the work order a random ID to store it under. - with datastore.describe(ReencryptionRequestModel, str(uuid.uuid4()), writeable=True) as new_request: - new_request.bob_verifying_key = bob_verifying_key - headers = {'Content-Type': 'application/octet-stream'} return Response(headers=headers, response=bytes(response)) diff --git a/nucypher/utilities/prometheus/collector.py b/nucypher/utilities/prometheus/collector.py index 59dfc6112..a4b069368 100644 --- a/nucypher/utilities/prometheus/collector.py +++ b/nucypher/utilities/prometheus/collector.py @@ -38,7 +38,6 @@ from nucypher.blockchain.eth.agents import ( ) from nucypher.blockchain.eth.interfaces import BlockchainInterfaceFactory from nucypher.blockchain.eth.registry import BaseContractRegistry -from nucypher.datastore.queries import get_reencryption_requests class MetricsCollector(ABC): @@ -129,11 +128,6 @@ class UrsulaInfoMetricsCollector(BaseMetricsCollector): self.metrics["learning_status"].state('running' if self.ursula._learning_task.running else 'stopped') self.metrics["known_nodes_gauge"].set(len(self.ursula.known_nodes)) - reencryption_requests = get_reencryption_requests(self.ursula.datastore) - self.metrics["reencryption_requests_gauge"].set( - len(reencryption_requests) if reencryption_requests else 0 - ) - if not self.ursula.federated_only: decentralized_payload = { "staking_provider_address": self.ursula.checksum_address, diff --git a/tests/acceptance/blockchain/actors/test_operator.py b/tests/acceptance/blockchain/actors/test_operator.py index c086c611c..aece1ca30 100644 --- a/tests/acceptance/blockchain/actors/test_operator.py +++ b/tests/acceptance/blockchain/actors/test_operator.py @@ -18,7 +18,6 @@ import pytest import pytest_twisted -from constant_sorrow.constants import MOCK_DB from twisted.internet import threads from twisted.internet.task import Clock from web3.middleware.simulate_unmined_transaction import unmined_receipt_simulator_middleware @@ -201,7 +200,6 @@ def test_ursula_operator_confirmation(ursula_decentralized_test_config, # make an ursula. blockchain_ursula = ursula_decentralized_test_config.produce( operator_address=operator_address, - db_filepath=MOCK_DB, rest_port=9151) # it's not confirmed @@ -258,7 +256,6 @@ def test_ursula_operator_confirmation_autopilot(mocker, # Make the Operator ursula = ursula_decentralized_test_config.produce( operator_address=operator2, - db_filepath=MOCK_DB, rest_port=9151) ursula.run(preflight=False, diff --git a/tests/acceptance/characters/test_ursula_web_status.py b/tests/acceptance/characters/test_ursula_web_status.py index b5c0fc27e..7b0d584dd 100644 --- a/tests/acceptance/characters/test_ursula_web_status.py +++ b/tests/acceptance/characters/test_ursula_web_status.py @@ -30,11 +30,8 @@ def ursula(blockchain_ursulas): @pytest.fixture(scope='module') def client(ursula): - db_fd, db_path = tempfile.mkstemp() - ursula.rest_app.config.update({"TESTING": True, "DATABASE": Path(db_path)}) + ursula.rest_app.config.update({"TESTING": True}) yield ursula.rest_app.test_client() - os.close(db_fd) - ursula.rest_app.config['DATABASE'].unlink() def test_ursula_html_renders(ursula, client): diff --git a/tests/conftest.py b/tests/conftest.py index 7965f01e0..67f6b3079 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,6 @@ along with nucypher. If not, see . from collections import defaultdict -import lmdb import pytest from eth_utils.crypto import keccak @@ -26,8 +25,7 @@ from nucypher.crypto.powers import TransactingPower from nucypher.network.nodes import Learner from nucypher.network.trackers import AvailabilityTracker from nucypher.utilities.logging import GlobalLoggerSettings -from tests.constants import INSECURE_DEVELOPMENT_PASSWORD, MOCK_IP_ADDRESS -from tests.mock.datastore import mock_lmdb_open +from tests.constants import MOCK_IP_ADDRESS # Crash on server error by default WebEmitter._crash_on_error_default = True @@ -183,12 +181,6 @@ def check_character_state_after_test(request): tracker.work_tracker.stop() -@pytest.fixture(scope='session', autouse=True) -def mock_datastore(monkeysession): - monkeysession.setattr(lmdb, 'open', mock_lmdb_open) - yield - - @pytest.fixture(scope='session', autouse=True) def mock_get_external_ip_from_url_source(session_mocker): target = 'nucypher.cli.actions.configure.determine_external_ip_address' diff --git a/tests/fixtures.py b/tests/fixtures.py index 9c22c5fd7..ee03183c0 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -63,7 +63,6 @@ from nucypher.config.constants import TEMPORARY_DOMAIN from nucypher.control.emitters import StdoutEmitter from nucypher.crypto.keystore import Keystore from nucypher.crypto.powers import TransactingPower -from nucypher.datastore import datastore from nucypher.network.nodes import TEACHER_NODES from nucypher.policy.conditions.context import USER_ADDRESS_CONTEXT from nucypher.utilities.logging import GlobalLoggerSettings, Logger @@ -139,11 +138,6 @@ def temp_dir_path(): temp_dir.cleanup() -@pytest.fixture(scope="module") -def test_datastore(): - test_datastore = datastore.Datastore(tempfile.mkdtemp()) - yield test_datastore - @pytest.fixture(scope='function') def certificates_tempdir(): diff --git a/tests/integration/config/test_character_configuration.py b/tests/integration/config/test_character_configuration.py index d7ebe1776..3b9b38ea9 100644 --- a/tests/integration/config/test_character_configuration.py +++ b/tests/integration/config/test_character_configuration.py @@ -15,7 +15,6 @@ along with nucypher. If not, see . """ -import tempfile from pathlib import Path import pytest @@ -175,7 +174,6 @@ def test_ursula_development_configuration(federated_only=True): # A Temporary Ursula port = ursula_one.rest_information()[0].port assert port == UrsulaConfiguration.DEFAULT_DEVELOPMENT_REST_PORT - assert tempfile.gettempdir() in str(ursula_one.datastore.db_path) assert ursula_one.certificate_filepath is CERTIFICATE_NOT_SAVED assert isinstance(ursula_one.node_storage, ForgetfulNodeStorage) assert ':memory:' in ursula_one.node_storage._name diff --git a/tests/integration/config/test_keystore_integration.py b/tests/integration/config/test_keystore_integration.py index b176bc16e..45af7dce9 100644 --- a/tests/integration/config/test_keystore_integration.py +++ b/tests/integration/config/test_keystore_integration.py @@ -29,7 +29,6 @@ from nucypher.characters.lawful import Alice, Bob, Ursula from nucypher.config.constants import TEMPORARY_DOMAIN from nucypher.crypto.keystore import Keystore from nucypher.crypto.powers import DecryptingPower, DelegatingPower, TLSHostingPower -from nucypher.datastore.datastore import Datastore from nucypher.network.server import ProxyRESTServer from nucypher.policy.payment import FreeReencryptions from nucypher.utilities.networking import LOOPBACK_ADDRESS @@ -130,6 +129,5 @@ def test_tls_hosting_certificate_remains_the_same(temp_dir_path, mocker): rest_host=LOOPBACK_ADDRESS, rest_port=rest_port, rest_app=IsType(Flask), - datastore=IsType(Datastore), hosting_power=tls_hosting_power) recreated_ursula.disenchant() diff --git a/tests/integration/learning/test_discovery_phases.py b/tests/integration/learning/test_discovery_phases.py index 67c5fc790..04d6fb002 100644 --- a/tests/integration/learning/test_discovery_phases.py +++ b/tests/integration/learning/test_discovery_phases.py @@ -18,23 +18,13 @@ along with nucypher. If not, see . import contextlib import time -from datetime import datetime -from unittest.mock import patch import maya -import pytest -from flask import Response - -from nucypher_core.umbral import SecretKey, Signer, PublicKey +from nucypher_core.umbral import SecretKey, Signer from nucypher.characters.lawful import Ursula from nucypher.crypto.signing import SignatureStamp -from nucypher.datastore.base import RecordField -from nucypher.network.nodes import Teacher -from tests.markers import skip_on_circleci from tests.mock.performance_mocks import ( - NotAPublicKey, - NotARestApp, VerificationTracker, mock_cert_loading, mock_cert_storage, @@ -43,7 +33,6 @@ from tests.mock.performance_mocks import ( mock_secret_source, mock_verify_node ) -from tests.utils.middleware import SluggishLargeFleetMiddleware from tests.utils.ursula import MOCK_KNOWN_URSULAS_CACHE """ diff --git a/tests/mock/performance_mocks.py b/tests/mock/performance_mocks.py index 03910d2a8..84e9e9aa0 100644 --- a/tests/mock/performance_mocks.py +++ b/tests/mock/performance_mocks.py @@ -145,14 +145,13 @@ class NotARestApp: _actual_rest_apps = [] _replaced_routes = {} - def __init__(self, this_node, db_filepath, *args, **kwargs): + def __init__(self, this_node, *args, **kwargs): self._actual_rest_app = None self.this_node = this_node - self.db_filepath = db_filepath @classmethod - def create_with_not_a_datastore(cls, *args, **kwargs): - return cls(*args, **kwargs), "this is not a datastore." + def create(cls, *args, **kwargs): + return cls(*args, **kwargs) @classmethod @contextmanager @@ -173,8 +172,7 @@ class NotARestApp: def actual_rest_app(self): if self._actual_rest_app is None: - self._actual_rest_app, self._datastore = make_rest_app(db_filepath=self.db_filepath, - this_node=self.this_node) + self._actual_rest_app = make_rest_app(this_node=self.this_node) _new_view_functions = self._ViewFunctions(self._actual_rest_app.view_functions) self._actual_rest_app.view_functions = _new_view_functions self._actual_rest_apps.append( @@ -201,7 +199,7 @@ class VerificationTracker: mock_cert_generation = patch("nucypher.crypto.tls.generate_self_signed_certificate", new=do_not_create_cert) mock_rest_app_creation = patch("nucypher.characters.lawful.make_rest_app", - new=NotARestApp.create_with_not_a_datastore) + new=NotARestApp.create) mock_remember_node = patch("nucypher.characters.lawful.Ursula.remember_node", new=simple_remember) mock_verify_node = patch("nucypher.characters.lawful.Ursula.verify_node", new=VerificationTracker.fake_verify_node) @@ -219,9 +217,9 @@ def mock_secret_source(*args, **kwargs): def _determine_good_serials(start, end): - ''' + """ Figure out which serials are good to use in mocks because they won't result in non-viable public keys. - ''' + """ good_serials = [] for i in range(start, end): try: diff --git a/tests/utils/ursula.py b/tests/utils/ursula.py index 966375b98..5639ac0a6 100644 --- a/tests/utils/ursula.py +++ b/tests/utils/ursula.py @@ -25,7 +25,6 @@ from nucypher.blockchain.eth.interfaces import BlockchainInterface from nucypher.characters.lawful import Ursula from nucypher.config.characters import UrsulaConfiguration from tests.constants import NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK -from tests.mock.datastore import MOCK_DB def select_test_port() -> int: @@ -61,7 +60,6 @@ def make_federated_ursulas(ursula_config: UrsulaConfiguration, federated_ursulas = set() for port in range(starting_port, starting_port+quantity): ursula = ursula_config.produce(rest_port=port + 100, - db_filepath=MOCK_DB, **ursula_overrides) federated_ursulas.add(ursula) @@ -96,7 +94,6 @@ def make_decentralized_ursulas(ursula_config: UrsulaConfiguration, for port, (staking_provider_address, operator_address) in enumerate(providers_and_operators, start=starting_port): ursula = ursula_config.produce(checksum_address=staking_provider_address, operator_address=operator_address, - db_filepath=MOCK_DB, rest_port=port + 100, **ursula_overrides)