Nucypher
/
nucypher
mirror of https://github.com/nucypher/nucypher.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Removes lmdb datastore usage

pull/2988/head
Kieran Prasch 2022-10-26 15:37:45 +01:00
parent 646246ae3f
commit febc6093c1
18 changed files with 23 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
examples/run_demo_ursula_fleet.py
View File

@ -65,12 +65,7 @@ def spin_up_federated_ursulas(quantity: int = FLEET_POPULATION):
deployer.start()
print(f"{u}: {deployer._listening_message()}")
try:
reactor.run()
finally:
for u in ursulas:
with suppress(FileNotFoundError):
shutil.rmtree(u.datastore.db_path)
reactor.run()
if __name__ == "__main__":

21
nucypher/characters/lawful.py
View File

@ -683,9 +683,7 @@ class Ursula(Teacher, Character, Operator):
self.payment_method = payment_method
# Server
self.rest_server = self._make_local_server(host=rest_host,
port=rest_port,
db_filepath=db_filepath)
self.rest_server = self._make_local_server(host=rest_host, port=rest_port)
# Self-signed TLS certificate of self for Teacher.__init__
certificate_filepath = self._crypto_power.power_ups(TLSHostingPower).keypair.certificate_filepath
@ -727,15 +725,11 @@ class Ursula(Teacher, Character, Operator):
self._crypto_power.consume_power_up(tls_hosting_power) # Consume!
return tls_hosting_power
def _make_local_server(self, host, port, db_filepath) -> ProxyRESTServer:
rest_app, datastore = make_rest_app(
this_node=self,
db_filepath=db_filepath,
)
def _make_local_server(self, host, port) -> ProxyRESTServer:
rest_app = make_rest_app(this_node=self)
rest_server = ProxyRESTServer(rest_host=host,
rest_port=port,
rest_app=rest_app,
datastore=datastore,
hosting_power=self.__get_hosting_power(host=host))
return rest_server
@ -899,9 +893,6 @@ class Ursula(Teacher, Character, Operator):
**Warning:** invalidates the Ursula.
"""
# `rest_server` holds references to the datastore (directly and via `rest_app`).
# An open datastore hogs up file descriptors.
self.rest_server = INVALIDATED
def rest_information(self):
@ -1083,12 +1074,6 @@ class Ursula(Teacher, Character, Operator):
#
# Properties
#
@property
def datastore(self):
try:
return self.rest_server.datastore
except AttributeError:
raise AttributeError("No rest server attached")
@property
def rest_url(self):

7
nucypher/cli/actions/confirm.py
View File

@ -57,16 +57,11 @@ def confirm_deployment(emitter: StdoutEmitter, deployer_interface: BlockchainDep
def confirm_destroy_configuration(config: CharacterConfiguration) -> bool:
"""Interactively confirm destruction of nucypher configuration files"""
# TODO: This is a workaround for ursula - needs follow up
try:
database = config.db_filepath
except AttributeError:
database = "No database found"
confirmation = CHARACTER_DESTRUCTION.format(name=config.NAME,
root=config.config_root,
keystore=config.keystore_dir,
nodestore=config.node_storage.source,
config=config.filepath,
database=database)
config=config.filepath)
click.confirm(confirmation, abort=True)
return True

1
nucypher/cli/literature.py
View File

@ -126,7 +126,6 @@ Delete all {name} character files including:
- Private and Public Keys ({keystore})
- Known Nodes ({nodestore})
- Node Configuration File ({config})
- Database ({database})
Are you sure?"""

1
nucypher/cli/options.py
View File

@ -41,7 +41,6 @@ from nucypher.utilities.logging import Logger
option_config_file = click.option('--config-file', help="Path to configuration file", type=EXISTING_READABLE_FILE)
option_config_root = click.option('--config-root', help="Custom configuration directory", type=click.Path(path_type=Path))
option_dev = click.option('--dev', '-d', help="Enable development mode", is_flag=True)
option_db_filepath = click.option('--db-filepath', help="The database filepath to connect to", type=click.Path(path_type=Path))
option_dry_run = click.option('--dry-run', '-x', help="Execute normally without actually starting the node", is_flag=True)
option_etherscan = click.option('--etherscan/--no-etherscan', help="Enable/disable viewing TX in Etherscan")
option_event_name = click.option('--event-name', help="Specify an event by name", type=click.STRING)

5
nucypher/cli/painting/nodes.py
View File

@ -17,7 +17,6 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
import maya
from nucypher.config.constants import SEEDNODES
from nucypher.datastore.queries import get_reencryption_requests
def build_fleet_state_status(ursula) -> str:
@ -41,9 +40,6 @@ def paint_node_status(emitter, ursula, start_time):
# Build FleetState status line
fleet_state = build_fleet_state_status(ursula=ursula)
reenc_requests = get_reencryption_requests(ursula.datastore)
num_reenc_requests = len(reenc_requests)
stats = ['⇀URSULA {}↽'.format(ursula.nickname.icon),
'{}'.format(ursula),
'Uptime .............. {}'.format(maya.now() - start_time),
@ -55,7 +51,6 @@ def paint_node_status(emitter, ursula, start_time):
'Rest Interface ...... {}'.format(ursula.rest_url()),
'Node Storage Type ... {}'.format(ursula.node_storage._name.capitalize()),
'Known Nodes ......... {}'.format(len(ursula.known_nodes)),
'Reencryption Requests {}'.format(num_reenc_requests),
teacher]
if not ursula.federated_only:

7
nucypher/config/characters.py
View File

@ -130,13 +130,6 @@ class UrsulaConfiguration(CharacterConfiguration):
merged_parameters = self.generate_parameters(**overrides)
ursula = self.CHARACTER_CLASS(**merged_parameters)
if self.dev_mode:
class MockDatastoreThreadPool(object):
def callInThread(self, f, *args, **kwargs):
return f(*args, **kwargs)
ursula.datastore_threadpool = MockDatastoreThreadPool()
return ursula
def destroy(self) -> None:

35
nucypher/network/server.py
View File

@ -37,8 +37,6 @@ from nucypher_core import (
from nucypher.config.constants import MAX_UPLOAD_CONTENT_LENGTH
from nucypher.crypto.keypairs import DecryptingKeypair
from nucypher.crypto.signing import InvalidSignature
from nucypher.datastore.datastore import Datastore
from nucypher.datastore.models import ReencryptionRequest as ReencryptionRequestModel
from nucypher.network.exceptions import NodeSeemsToBeDown
from nucypher.network.nodes import NodeSprout
from nucypher.network.protocols import InterfaceInfo
@ -65,14 +63,12 @@ class ProxyRESTServer:
rest_host: str,
rest_port: int,
hosting_power=None,
rest_app=None,
datastore=None,
rest_app=None
) -> None:
self.rest_interface = InterfaceInfo(host=rest_host, port=rest_port)
if rest_app: # if is me
self.rest_app = rest_app
self.datastore = datastore
else:
self.rest_app = constants.PUBLIC_ONLY
@ -83,30 +79,20 @@ class ProxyRESTServer:
def make_rest_app(
db_filepath: Path,
this_node,
log: Logger = Logger("http-application-layer")
) -> Tuple[Flask, Datastore]:
"""
Creates a REST application and an associated ``Datastore`` object.
Note that the REST app **does not** hold a reference to the datastore;
it is your responsibility to ensure it lives for as long as the app does.
"""
) -> Flask:
"""Creates a REST application."""
# A trampoline function for the real REST app,
# to ensure that a reference to the node and the datastore object is not held by the app closure.
# to ensure that a reference to the node object is not held by the app closure.
# One would think that it's enough to only remove a reference to the node,
# but `rest_app` somehow holds a reference to itself, Uroboros-like,
# and will hold the datastore reference if it is created there.
log.info("Starting datastore {}".format(db_filepath))
datastore = Datastore(db_filepath)
rest_app = _make_rest_app(weakref.proxy(datastore), weakref.proxy(this_node), log)
return rest_app, datastore
# but `rest_app` somehow holds a reference to itself, Uroboros-like...
rest_app = _make_rest_app(weakref.proxy(this_node), log)
return rest_app
def _make_rest_app(datastore: Datastore, this_node, log: Logger) -> Flask:
def _make_rest_app(this_node, log: Logger) -> Flask:
# TODO: Avoid circular imports :-(
from nucypher.characters.lawful import Alice, Bob, Ursula
@ -292,11 +278,6 @@ def _make_rest_app(datastore: Datastore, this_node, log: Logger) -> Flask:
# TODO: return a sensible response if it fails (currently results in 500)
response = this_node._reencrypt(kfrag=verified_kfrag, capsules=capsules_to_process)
# Now, Ursula saves evidence of this workorder to her database...
# Note: we give the work order a random ID to store it under.
with datastore.describe(ReencryptionRequestModel, str(uuid.uuid4()), writeable=True) as new_request:
new_request.bob_verifying_key = bob_verifying_key
headers = {'Content-Type': 'application/octet-stream'}
return Response(headers=headers, response=bytes(response))

6
nucypher/utilities/prometheus/collector.py
View File

@ -38,7 +38,6 @@ from nucypher.blockchain.eth.agents import (
)
from nucypher.blockchain.eth.interfaces import BlockchainInterfaceFactory
from nucypher.blockchain.eth.registry import BaseContractRegistry
from nucypher.datastore.queries import get_reencryption_requests
class MetricsCollector(ABC):
@ -129,11 +128,6 @@ class UrsulaInfoMetricsCollector(BaseMetricsCollector):
self.metrics["learning_status"].state('running' if self.ursula._learning_task.running else 'stopped')
self.metrics["known_nodes_gauge"].set(len(self.ursula.known_nodes))
reencryption_requests = get_reencryption_requests(self.ursula.datastore)
self.metrics["reencryption_requests_gauge"].set(
len(reencryption_requests) if reencryption_requests else 0
)
if not self.ursula.federated_only:
decentralized_payload = {
"staking_provider_address": self.ursula.checksum_address,

3
tests/acceptance/blockchain/actors/test_operator.py
View File

@ -18,7 +18,6 @@
import pytest
import pytest_twisted
from constant_sorrow.constants import MOCK_DB
from twisted.internet import threads
from twisted.internet.task import Clock
from web3.middleware.simulate_unmined_transaction import unmined_receipt_simulator_middleware
@ -201,7 +200,6 @@ def test_ursula_operator_confirmation(ursula_decentralized_test_config,
# make an ursula.
blockchain_ursula = ursula_decentralized_test_config.produce(
operator_address=operator_address,
db_filepath=MOCK_DB,
rest_port=9151)
# it's not confirmed
@ -258,7 +256,6 @@ def test_ursula_operator_confirmation_autopilot(mocker,
# Make the Operator
ursula = ursula_decentralized_test_config.produce(
operator_address=operator2,
db_filepath=MOCK_DB,
rest_port=9151)
ursula.run(preflight=False,

5
tests/acceptance/characters/test_ursula_web_status.py
View File

@ -30,11 +30,8 @@ def ursula(blockchain_ursulas):
@pytest.fixture(scope='module')
def client(ursula):
db_fd, db_path = tempfile.mkstemp()
ursula.rest_app.config.update({"TESTING": True, "DATABASE": Path(db_path)})
ursula.rest_app.config.update({"TESTING": True})
yield ursula.rest_app.test_client()
os.close(db_fd)
ursula.rest_app.config['DATABASE'].unlink()
def test_ursula_html_renders(ursula, client):

10
tests/conftest.py
View File

@ -17,7 +17,6 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
from collections import defaultdict
import lmdb
import pytest
from eth_utils.crypto import keccak
@ -26,8 +25,7 @@ from nucypher.crypto.powers import TransactingPower
from nucypher.network.nodes import Learner
from nucypher.network.trackers import AvailabilityTracker
from nucypher.utilities.logging import GlobalLoggerSettings
from tests.constants import INSECURE_DEVELOPMENT_PASSWORD, MOCK_IP_ADDRESS
from tests.mock.datastore import mock_lmdb_open
from tests.constants import MOCK_IP_ADDRESS
# Crash on server error by default
WebEmitter._crash_on_error_default = True
@ -183,12 +181,6 @@ def check_character_state_after_test(request):
tracker.work_tracker.stop()
@pytest.fixture(scope='session', autouse=True)
def mock_datastore(monkeysession):
monkeysession.setattr(lmdb, 'open', mock_lmdb_open)
yield
@pytest.fixture(scope='session', autouse=True)
def mock_get_external_ip_from_url_source(session_mocker):
target = 'nucypher.cli.actions.configure.determine_external_ip_address'

6
tests/fixtures.py
View File

@ -63,7 +63,6 @@ from nucypher.config.constants import TEMPORARY_DOMAIN
from nucypher.control.emitters import StdoutEmitter
from nucypher.crypto.keystore import Keystore
from nucypher.crypto.powers import TransactingPower
from nucypher.datastore import datastore
from nucypher.network.nodes import TEACHER_NODES
from nucypher.policy.conditions.context import USER_ADDRESS_CONTEXT
from nucypher.utilities.logging import GlobalLoggerSettings, Logger
@ -139,11 +138,6 @@ def temp_dir_path():
temp_dir.cleanup()
@pytest.fixture(scope="module")
def test_datastore():
test_datastore = datastore.Datastore(tempfile.mkdtemp())
yield test_datastore
@pytest.fixture(scope='function')
def certificates_tempdir():

2
tests/integration/config/test_character_configuration.py
View File

@ -15,7 +15,6 @@
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
import tempfile
from pathlib import Path
import pytest
@ -175,7 +174,6 @@ def test_ursula_development_configuration(federated_only=True):
# A Temporary Ursula
port = ursula_one.rest_information()[0].port
assert port == UrsulaConfiguration.DEFAULT_DEVELOPMENT_REST_PORT
assert tempfile.gettempdir() in str(ursula_one.datastore.db_path)
assert ursula_one.certificate_filepath is CERTIFICATE_NOT_SAVED
assert isinstance(ursula_one.node_storage, ForgetfulNodeStorage)
assert ':memory:' in ursula_one.node_storage._name

2
tests/integration/config/test_keystore_integration.py
View File

@ -29,7 +29,6 @@ from nucypher.characters.lawful import Alice, Bob, Ursula
from nucypher.config.constants import TEMPORARY_DOMAIN
from nucypher.crypto.keystore import Keystore
from nucypher.crypto.powers import DecryptingPower, DelegatingPower, TLSHostingPower
from nucypher.datastore.datastore import Datastore
from nucypher.network.server import ProxyRESTServer
from nucypher.policy.payment import FreeReencryptions
from nucypher.utilities.networking import LOOPBACK_ADDRESS
@ -130,6 +129,5 @@ def test_tls_hosting_certificate_remains_the_same(temp_dir_path, mocker):
rest_host=LOOPBACK_ADDRESS,
rest_port=rest_port,
rest_app=IsType(Flask),
datastore=IsType(Datastore),
hosting_power=tls_hosting_power)
recreated_ursula.disenchant()

13
tests/integration/learning/test_discovery_phases.py
View File

@ -18,23 +18,13 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
import contextlib
import time
from datetime import datetime
from unittest.mock import patch
import maya
import pytest
from flask import Response
from nucypher_core.umbral import SecretKey, Signer, PublicKey
from nucypher_core.umbral import SecretKey, Signer
from nucypher.characters.lawful import Ursula
from nucypher.crypto.signing import SignatureStamp
from nucypher.datastore.base import RecordField
from nucypher.network.nodes import Teacher
from tests.markers import skip_on_circleci
from tests.mock.performance_mocks import (
NotAPublicKey,
NotARestApp,
VerificationTracker,
mock_cert_loading,
mock_cert_storage,
@ -43,7 +33,6 @@ from tests.mock.performance_mocks import (
mock_secret_source,
mock_verify_node
)
from tests.utils.middleware import SluggishLargeFleetMiddleware
from tests.utils.ursula import MOCK_KNOWN_URSULAS_CACHE
"""

16
tests/mock/performance_mocks.py
View File

@ -145,14 +145,13 @@ class NotARestApp:
_actual_rest_apps = []
_replaced_routes = {}
def __init__(self, this_node, db_filepath, *args, **kwargs):
def __init__(self, this_node, *args, **kwargs):
self._actual_rest_app = None
self.this_node = this_node
self.db_filepath = db_filepath
@classmethod
def create_with_not_a_datastore(cls, *args, **kwargs):
return cls(*args, **kwargs), "this is not a datastore."
def create(cls, *args, **kwargs):
return cls(*args, **kwargs)
@classmethod
@contextmanager
@ -173,8 +172,7 @@ class NotARestApp:
def actual_rest_app(self):
if self._actual_rest_app is None:
self._actual_rest_app, self._datastore = make_rest_app(db_filepath=self.db_filepath,
this_node=self.this_node)
self._actual_rest_app = make_rest_app(this_node=self.this_node)
_new_view_functions = self._ViewFunctions(self._actual_rest_app.view_functions)
self._actual_rest_app.view_functions = _new_view_functions
self._actual_rest_apps.append(
@ -201,7 +199,7 @@ class VerificationTracker:
mock_cert_generation = patch("nucypher.crypto.tls.generate_self_signed_certificate", new=do_not_create_cert)
mock_rest_app_creation = patch("nucypher.characters.lawful.make_rest_app",
new=NotARestApp.create_with_not_a_datastore)
new=NotARestApp.create)
mock_remember_node = patch("nucypher.characters.lawful.Ursula.remember_node", new=simple_remember)
mock_verify_node = patch("nucypher.characters.lawful.Ursula.verify_node", new=VerificationTracker.fake_verify_node)
@ -219,9 +217,9 @@ def mock_secret_source(*args, **kwargs):
def _determine_good_serials(start, end):
'''
"""
Figure out which serials are good to use in mocks because they won't result in non-viable public keys.
'''
"""
good_serials = []
for i in range(start, end):
try:

3
tests/utils/ursula.py
View File

@ -25,7 +25,6 @@ from nucypher.blockchain.eth.interfaces import BlockchainInterface
from nucypher.characters.lawful import Ursula
from nucypher.config.characters import UrsulaConfiguration
from tests.constants import NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK
from tests.mock.datastore import MOCK_DB
def select_test_port() -> int:
@ -61,7 +60,6 @@ def make_federated_ursulas(ursula_config: UrsulaConfiguration,
federated_ursulas = set()
for port in range(starting_port, starting_port+quantity):
ursula = ursula_config.produce(rest_port=port + 100,
db_filepath=MOCK_DB,
**ursula_overrides)
federated_ursulas.add(ursula)
@ -96,7 +94,6 @@ def make_decentralized_ursulas(ursula_config: UrsulaConfiguration,
for port, (staking_provider_address, operator_address) in enumerate(providers_and_operators, start=starting_port):
ursula = ursula_config.produce(checksum_address=staking_provider_address,
operator_address=operator_address,
db_filepath=MOCK_DB,
rest_port=port + 100,
**ursula_overrides)