Merge pull request #665 from KPrasch/fix-demo

Heartbeat Demo Compatibility: Storage Defaults, File-based Temp DB

examples/heartbeat_demo/alicia.py

@@ -23,12 +23,10 @@ POLICY_FILENAME = "policy-metadata.json"
 # # Twisted Logger
 globalLogPublisher.addObserver(SimpleObserver())
 #
-# # Temporary file storage
+TEMP_ALICE_DIR = "alicia-files".format(os.path.dirname(os.path.abspath(__file__)))
-TEMP_ALICE_DIR = "{}/alicia-files".format(os.path.dirname(os.path.abspath(__file__)))
-TEMP_URSULA_CERTIFICATE_DIR = "{}/ursula-certs".format(TEMP_ALICE_DIR)
 
 # We expect the url of the seednode as the first argument.
-SEEDNODE_URL = sys.argv[1]
+SEEDNODE_URL = 'localhost:11500'
 
 
 #######################################
@@ -37,9 +35,11 @@ SEEDNODE_URL = sys.argv[1]
 
 
 # We get a persistent Alice.
+# If we had an existing Alicia in disk, let's get it from there
+
 passphrase = "TEST_ALICIA_INSECURE_DEVELOPMENT_PASSWORD"
-try:  # If we had an existing Alicia in disk, let's get it from there
+try:
-    alice_config_file = os.path.join(TEMP_ALICE_DIR, "config_root", "alice.config")
+    alice_config_file = os.path.join(TEMP_ALICE_DIR, "alice.config")
     new_alice_config = AliceConfiguration.from_configuration_file(
             filepath=alice_config_file,
             network_middleware=RestMiddleware(),
@@ -47,25 +47,29 @@ try:  # If we had an existing Alicia in disk, let's get it from there
             save_metadata=False,
         )
     alicia = new_alice_config(passphrase=passphrase)
-except:  # If anything fails, let's create Alicia from scratch
+
+except:
+
+    # If anything fails, let's create Alicia from scratch
     # Remove previous demo files and create new ones
+
     shutil.rmtree(TEMP_ALICE_DIR, ignore_errors=True)
-    os.mkdir(TEMP_ALICE_DIR)
-    os.mkdir(TEMP_URSULA_CERTIFICATE_DIR)
 
     ursula = Ursula.from_seed_and_stake_info(seed_uri=SEEDNODE_URL,
                                              federated_only=True,
                                              minimum_stake=0)
 
     alice_config = AliceConfiguration(
-        config_root=os.path.join(TEMP_ALICE_DIR, "config_root"),
+        config_root=os.path.join(TEMP_ALICE_DIR),
         is_me=True,
         known_nodes={ursula},
         start_learning_now=False,
         federated_only=True,
         learn_on_same_thread=True,
     )
+
     alice_config.initialize(password=passphrase)
+
     alice_config.keyring.unlock(password=passphrase)
     alicia = alice_config.produce()
 

examples/heartbeat_demo/doctor.py

@@ -9,9 +9,9 @@ from timeit import default_timer as timer
 
 from nucypher.characters.lawful import Bob, Ursula
 from nucypher.crypto.kits import UmbralMessageKit
-from nucypher.crypto.powers import EncryptingPower, SigningPower
+from nucypher.crypto.powers import DecryptingPower, SigningPower
 from nucypher.data_sources import DataSource
-from nucypher.keystore.keypairs import EncryptingKeypair, SigningKeypair
+from nucypher.keystore.keypairs import DecryptingKeypair, SigningKeypair
 from nucypher.network.middleware import RestMiddleware
 
 from umbral.keys import UmbralPublicKey
@@ -21,22 +21,15 @@ from umbral.keys import UmbralPublicKey
 # Boring setup stuff #
 ######################
 
-SEEDNODE_URL = sys.argv[1]
+SEEDNODE_URL = 'localhost:11501'
 
 # TODO: path joins?
 TEMP_DOCTOR_DIR = "{}/doctor-files".format(os.path.dirname(os.path.abspath(__file__)))
 
-TEMP_URSULA_CERTIFICATE_DIR = "{}/ursula-certs".format(TEMP_DOCTOR_DIR)
-TEMP_DOCTOR_CERTIFICATE_DIR = "{}/doctor-certs".format(TEMP_DOCTOR_DIR)
-
 # Remove previous demo files and create new ones
 shutil.rmtree(TEMP_DOCTOR_DIR, ignore_errors=True)
-os.mkdir(TEMP_DOCTOR_DIR)
-os.mkdir(TEMP_URSULA_CERTIFICATE_DIR)
-os.mkdir(TEMP_DOCTOR_CERTIFICATE_DIR)
 
-ursula = Ursula.from_seed_and_stake_info(host=SEEDNODE_URL,
+ursula = Ursula.from_seed_and_stake_info(seed_uri=SEEDNODE_URL,
-                                         certificates_directory=TEMP_URSULA_CERTIFICATE_DIR,
                                          federated_only=True,
                                          minimum_stake=0)
 
@@ -44,9 +37,9 @@ ursula = Ursula.from_seed_and_stake_info(host=SEEDNODE_URL,
 from doctor_keys import get_doctor_privkeys
 doctor_keys = get_doctor_privkeys()
 
-bob_enc_keypair = EncryptingKeypair(private_key=doctor_keys["enc"])
+bob_enc_keypair = DecryptingKeypair(private_key=doctor_keys["enc"])
 bob_sig_keypair = SigningKeypair(private_key=doctor_keys["sig"])
-enc_power = EncryptingPower(keypair=bob_enc_keypair)
+enc_power = DecryptingPower(keypair=bob_enc_keypair)
 sig_power = SigningPower(keypair=bob_sig_keypair)
 power_ups = [enc_power, sig_power]
 
@@ -57,7 +50,6 @@ doctor = Bob(
     federated_only=True,
     crypto_power_ups=power_ups,
     start_learning_now=True,
-    known_certificates_dir=TEMP_DOCTOR_CERTIFICATE_DIR,
     abort_on_learning_error=True,
     known_nodes=[ursula],
     save_metadata=False,

nucypher/characters/lawful.py

@@ -707,7 +707,7 @@ class Ursula(Teacher, Character, Miner):
                                  seed_uri: str,
                                  federated_only: bool,
                                  minimum_stake: int = 0,
-                                 checksum_address: str = None,
+                                 checksum_address: str = None,   # TODO: Why is this unused?
                                  network_middleware: RestMiddleware = None,
                                  *args,
                                  **kwargs

nucypher/cli/main.py

@@ -17,16 +17,17 @@ along with nucypher.  If not, see <https://www.gnu.org/licenses/>.
 """
 
 import os
+import shutil
 
 import click
-import shutil
-from constant_sorrow.constants import NO_BLOCKCHAIN_CONNECTION, NO_PASSWORD
-from constant_sorrow.constants import TEMPORARY_DOMAIN
 from nacl.exceptions import CryptoError
 from twisted.internet import stdio
 from twisted.logger import Logger
 from twisted.logger import globalLogPublisher
 
+from constant_sorrow import constants
+from constant_sorrow.constants import NO_BLOCKCHAIN_CONNECTION, NO_PASSWORD
+from constant_sorrow.constants import TEMPORARY_DOMAIN
 from nucypher.blockchain.eth.constants import MIN_LOCKED_PERIODS, MAX_MINTING_PERIODS
 from nucypher.blockchain.eth.registry import EthereumContractRegistry
 from nucypher.characters.lawful import Ursula
@@ -341,9 +342,15 @@ Delete {}?'''.format(ursula_config.config_root), abort=True)
     # Authenticated Configurations
     else:
 
-        # Restore configuration from file
+        # Deserialize network domain name if override passed
+        if network:
+            domain_constant = getattr(constants, network.upper())
+            domains = {domain_constant}
+        else:
+            domains = None
+
         ursula_config = UrsulaConfiguration.from_configuration_file(filepath=config_file,
-                                                                    domains={network} if network else None,
+                                                                    domains=domains,
                                                                     registry_filepath=registry_filepath,
                                                                     provider_uri=provider_uri,
                                                                     rest_host=rest_host,

nucypher/config/characters.py

@@ -21,7 +21,6 @@ import os
 from constant_sorrow.constants import (
     UNINITIALIZED_CONFIGURATION
 )
-
 from nucypher.config.constants import DEFAULT_CONFIG_ROOT
 from nucypher.config.keyring import NucypherKeyring
 from nucypher.config.node import NodeConfiguration
@@ -41,8 +40,6 @@ class UrsulaConfiguration(NodeConfiguration):
                  dev_mode: bool = False,
                  db_filepath: str = None,
                  *args, **kwargs) -> None:
-        if dev_mode is True:
-            db_filepath = ':memory:'  # sqlite in-memory db
         self.db_filepath = db_filepath or UNINITIALIZED_CONFIGURATION
         super().__init__(dev_mode=dev_mode, *args, **kwargs)
 

nucypher/config/node.py

@@ -20,12 +20,20 @@ import binascii
 import json
 import os
 import secrets
+import shutil
 import string
 from abc import ABC
 from json import JSONDecodeError
 from tempfile import TemporaryDirectory
+from typing import List, Set
 
-import shutil
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurve
+from cryptography.x509 import Certificate
+from twisted.logger import Logger
+from umbral.signing import Signature
+
+from constant_sorrow import constants
 from constant_sorrow.constants import GLOBAL_DOMAIN
 from constant_sorrow.constants import (
     UNINITIALIZED_CONFIGURATION,
@@ -34,13 +42,6 @@ from constant_sorrow.constants import (
     LIVE_CONFIGURATION,
     NO_KEYRING_ATTACHED
 )
-from cryptography.hazmat.primitives.asymmetric import ec
-from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurve
-from cryptography.x509 import Certificate
-from twisted.logger import Logger
-from typing import List, Set
-from umbral.signing import Signature
-
 from nucypher.blockchain.eth.agents import PolicyAgent, MinerAgent, NucypherTokenAgent
 from nucypher.blockchain.eth.chains import Blockchain
 from nucypher.config.constants import DEFAULT_CONFIG_ROOT, BASE_DIR, USER_LOG_DIR
@@ -64,6 +65,8 @@ class NodeConfiguration(ABC):
 
     # Mode
     DEFAULT_OPERATING_MODE = 'decentralized'
+
+    # Domains
     DEFAULT_DOMAIN = GLOBAL_DOMAIN
 
     # Serializers
@@ -76,7 +79,7 @@ class NodeConfiguration(ABC):
     TEMP_CONFIGURATION_DIR_PREFIX = "nucypher-tmp-"
 
     # Blockchain
-    DEFAULT_PROVIDER_URI = 'tester://pyevm'  # FIXME: Needs to be updated in tandem with manual providers for interface.connect
+    DEFAULT_PROVIDER_URI = 'tester://pyevm'
 
     # Registry
     __REGISTRY_NAME = 'contract_registry.json'
@@ -371,7 +374,10 @@ class NodeConfiguration(ABC):
                                                   serializer=cls.NODE_SERIALIZER,
                                                   deserializer=cls.NODE_DESERIALIZER)
 
-        payload.update(dict(node_storage=node_storage, domains=set(payload['domains'])))
+        # Deserialize domains to Constants vis Constant Sorrow
+        domains = set(getattr(constants, domain.upper()) for domain in payload['domains'])
+
+        payload.update(dict(node_storage=node_storage, domains=domains))
 
         # Filter out Nones from overrides to detect, well, overrides
         overrides = {k: v for k, v in overrides.items() if v is not None}
@@ -391,7 +397,7 @@ class NodeConfiguration(ABC):
         del payload['is_me']  # TODO
 
         # Serialize domains
-        domains = list(str(d) for d in self.domains)
+        domains = list(str(domain) for domain in self.domains)
 
         # Save node connection data
         payload.update(dict(node_storage=self.node_storage.payload(), domains=domains))

nucypher/network/nodes.py

@@ -302,9 +302,7 @@ class Learner:
 
     def load_seednodes(self,
                        read_storages: bool = True,
-                       retry_attempts: int = 3,
+                       retry_attempts: int = 3):  # TODO: why are these unused?
-                       retry_rate: int = 2,
-                       timeout=3):  # TODO: why are these unused?
         """
         Engage known nodes from storages and pre-fetch hardcoded seednode certificates for node learning.
         """
@@ -312,8 +310,9 @@ class Learner:
             self.log.debug("Already done seeding; won't try again.")
             return
 
-        def __attempt_seednode_learning(seednode_metadata, current_attempt=1):
+        from nucypher.characters.lawful import Ursula
-            from nucypher.characters.lawful import Ursula
+        for seednode_metadata in self._seed_nodes:
+
             self.log.debug(
                 "Seeding from: {}|{}:{}".format(seednode_metadata.checksum_public_address,
                                                 seednode_metadata.rest_host,
@@ -328,17 +327,15 @@ class Learner:
                 self.unresponsive_seed_nodes.discard(seednode_metadata)
                 self.remember_node(seed_node)
 
-        for seednode_metadata in self._seed_nodes:
+        if not self.unresponsive_seed_nodes:
-            __attempt_seednode_learning(seednode_metadata=seednode_metadata)
-
-        if not self.unresponsive_seed_nodes and not self.lonely:
             self.log.info("Finished learning about all seednodes.")
+
         self.done_seeding = True
 
         if read_storages is True:
             self.read_nodes_from_storage()
 
-        if not self.known_nodes and not self.lonely:
+        if not self.known_nodes:
             self.log.warn("No seednodes were available after {} attempts".format(retry_attempts))
             # TODO: Need some actual logic here for situation with no seed nodes (ie, maybe try again much later)
 
@@ -401,8 +398,14 @@ class Learner:
             return False
         elif now:
             self.log.info("Starting Learning Loop NOW.")
-            if not self.lonely:
+
+            if self.lonely:
+                self.done_seeding = True
+                self.read_nodes_from_storage()
+
+            else:
                 self.load_seednodes()
+
             self.learn_from_teacher_node()
             self.learning_deferred = self._learning_task.start(interval=self._SHORT_LEARNING_DELAY)
             self.learning_deferred.addErrback(self.handle_learning_errors)
@@ -458,7 +461,7 @@ class Learner:
         self.teacher_nodes.extend(nodes_we_know_about)
 
     def cycle_teacher_node(self):
-        # To ensure that all the best teachers are availalble, first let's make sure
+        # To ensure that all the best teachers are available, first let's make sure
         # that we have connected to all the seed nodes.
         if self.unresponsive_seed_nodes and not self.lonely:
             self.log.info("Still have unresponsive seed nodes; trying again to connect.")
@@ -666,7 +669,8 @@ class Learner:
             # In this case, this node knows about no other nodes.  Hopefully we've taught it something.
             if response.content == b"":
                 return NO_KNOWN_NODES
-            # In the other case - where the status code is 204 but the repsonse isn't blank - we'll keep parsing.  It's possible that our fleet states match, and we'll check for that later.
+            # In the other case - where the status code is 204 but the repsonse isn't blank - we'll keep parsing.
+            # It's possible that our fleet states match, and we'll check for that later.
 
         elif response.status_code != 200:
             self.log.info("Bad response from teacher {}: {} - {}".format(current_teacher, response, response.content))

nucypher/network/server.py

@@ -147,8 +147,12 @@ class ProxyRESTRoutes:
         self.log.info("Starting datastore {}".format(self.db_filepath))
 
         # See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings
-        db_filepath = (self.db_filepath or '')  # Capture None
+        if self.db_filepath:
-        engine = create_engine('sqlite:///{}'.format(db_filepath))
+            db_uri = f'sqlite:///{self.db_filepath}'
+        else:
+            db_uri = 'sqlite://'   # TODO: Is this a sane default? See #667
+
+        engine = create_engine(db_uri)
 
         Base.metadata.create_all(engine)
         self.datastore = keystore.KeyStore(engine)

tests/cli/commands/test_run_ursula.py

@@ -42,7 +42,6 @@ def test_run_lone_federated_default_development_ursula(click_runner):
 
     time.sleep(Learner._SHORT_LEARNING_DELAY)
     assert result.exit_code == 0
-    assert ":memory:" in result.output
     assert "Running Ursula on 127.0.0.1:{}".format(MOCK_URSULA_STARTING_PORT)
 
     reserved_ports = (NodeConfiguration.DEFAULT_REST_PORT, NodeConfiguration.DEFAULT_DEVELOPMENT_REST_PORT)

tests/config/test_ursula_config.py

@@ -25,7 +25,7 @@ def test_ursula_development_configuration(federated_only=True):
     # A Temporary Ursula
     port = ursula_one.rest_information()[0].port
     assert port == UrsulaConfiguration.DEFAULT_DEVELOPMENT_REST_PORT
-    assert ursula_one.datastore.engine.url.database == ":memory:"
+    assert '/tmp' in ursula_one.datastore.engine.url.database
     assert ursula_one.certificate_filepath is CERTIFICATE_NOT_SAVED
     assert UrsulaConfiguration.TEMP_CONFIGURATION_DIR_PREFIX in ursula_one.keyring_dir
     assert isinstance(ursula_one.node_storage, ForgetfulNodeStorage)