Cleanup Ursula stopping services, prevent halting of reactor via health checks in test; test touch ups.

nucypher/blockchain/eth/registry.py

@@ -378,7 +378,7 @@ class LocalContractRegistry(BaseContractRegistry):
         """
         try:
             with open(self.filepath, 'r') as registry_file:
-                self.log.debug("Reading from registrar: filepath {}".format(self.filepath))
+                self.log.debug("Reading from registry: filepath {}".format(self.filepath))
                 registry_file.seek(0)
                 file_data = registry_file.read()
                 if file_data:

nucypher/blockchain/eth/token.py

@@ -510,8 +510,9 @@ class WorkTracker:
         return self.__current_period
 
     def stop(self) -> None:
-        self._tracking_task.stop()
-        self.log.info(f"STOPPED WORK TRACKING")
+        if self._tracking_task.running:
+            self._tracking_task.stop()
+            self.log.info(f"STOPPED WORK TRACKING")
 
     def start(self, act_now: bool = False, requirement_func: Callable = None, force: bool = False) -> None:
         """

nucypher/characters/lawful.py

@@ -967,21 +967,20 @@ class Ursula(Teacher, Character, Worker):
 
         if is_me:
 
-            # Learner
-            self._start_learning_now = start_learning_now
-            self.known_nodes.record_fleet_state(additional_nodes_to_track=[self])  # Initial Impression
-
             # In-Memory TreasureMap tracking
             self._stored_treasure_maps = dict()
 
+            # Learner
+            self._start_learning_now = start_learning_now
+
             # Self-Health Checks
             self._availability_check = availability_check
             self._availability_sensor = AvailabilitySensor(ursula=self)
 
             # Arrangement Pruning
+            self.__pruning_task = None
             self._prune_datastore = prune_datastore
             self._arrangement_pruning_task = LoopingCall(f=self.__prune_arrangements)
-            self.__pruning_task = None
 
             # Prometheus / Metrics
             self._metrics_port = metrics_port
@@ -1079,6 +1078,8 @@ class Ursula(Teacher, Character, Worker):
                          decentralized_identity_evidence=decentralized_identity_evidence)
 
         if is_me:
+            self.known_nodes.record_fleet_state(additional_nodes_to_track=[self])  # Initial Impression
+
             message = "THIS IS YOU: {}: {}".format(self.__class__.__name__, self)
             self.log.info(message)
             self.log.info(self.banner.format(self.nickname))
@@ -1121,7 +1122,6 @@ class Ursula(Teacher, Character, Worker):
         if learning:
             if emitter:
                 emitter.message(f"Connecting to {','.join(self.learning_domains)}", color='green', bold=True)
-            # Initial Fleet State
             self.start_learning_loop(now=self._start_learning_now)
 
         if availability:
@@ -1169,6 +1169,18 @@ class Ursula(Teacher, Character, Worker):
         elif start_reactor:  # ... without hendrix
             reactor.run()    # <--- Blocking Call (Reactor)
 
+    def stop(self, halt_reactor: bool = False) -> None:
+        """Stop services"""
+        self._availability_sensor.stop()
+        if self._learning_task.running:
+            self.stop_learning_loop()
+        if not self.federated_only:
+            self.work_tracker.stop()
+        if self._arrangement_pruning_task.running:
+            self._arrangement_pruning_task.stop()
+        if halt_reactor:
+            reactor.stop()
+
     def rest_information(self):
         hosting_power = self._crypto_power.power_ups(TLSHostingPower)
 

nucypher/network/sensors.py

@@ -11,7 +11,7 @@ from nucypher.network.middleware import RestMiddleware
 
 class AvailabilitySensor:
 
-    FAST_INTERVAL = 5          # Seconds
+    FAST_INTERVAL = 15          # Seconds
     SLOW_INTERVAL = 60 * 5
     SEEDING_DURATION = 60
     MAXIMUM_ALONE_TIME = 120
@@ -61,14 +61,21 @@ class AvailabilitySensor:
                       f'Please check your network and firewall configuration.'
                       f'Auto-shutdown will commence soon if the services do not become available.')
 
-    def shutdown_everything(self, reason = None):
+    def shutdown_everything(self, reason=None, halt_reactor=True):
+        self.log.warn(f'[NODE IS UNREACHABLE] Commencing auto-shutdown sequence...')
+        self._ursula.stop(halt_reactor=False)
         try:
             if reason:
                 raise reason(reason.message)
             raise self.Unreachable(f'{self._ursula} is unreachable.')
         finally:
-            if reactor.running:
-                reactor.stop()
+            if halt_reactor:
+                self._halt_reactor()
+
+    @staticmethod
+    def _halt_reactor() -> None:
+        if reactor.running:
+            reactor.stop()
 
     def handle_measurement_errors(self, *args, **kwargs) -> None:
         failure = args[0]

scripts/circle/logOutput.sh

@@ -2,7 +2,7 @@
 
 LOGDIR=/tmp/ursulas-logs
 mkdir $LOGDIR
-docker exec -it circleusula1 cat /root/.cache/nucypher/log/nucypher.log > $LOGDIR/ursula-1.txt
-docker exec -it circleusula2 cat /root/.cache/nucypher/log/nucypher.log > $LOGDIR/ursula-2.txt
-docker exec -it circleusula3 cat /root/.cache/nucypher/log/nucypher.log > $LOGDIR/ursula-3.txt
-docker exec -it circleusula4 cat /root/.cache/nucypher/log/nucypher.log > $LOGDIR/ursula-4.txt
+docker exec -it circleursula1 cat /root/.cache/nucypher/log/nucypher.log > $LOGDIR/ursula-1.txt
+docker exec -it circleursula2 cat /root/.cache/nucypher/log/nucypher.log > $LOGDIR/ursula-2.txt
+docker exec -it circleursula3 cat /root/.cache/nucypher/log/nucypher.log > $LOGDIR/ursula-3.txt
+docker exec -it circleursula4 cat /root/.cache/nucypher/log/nucypher.log > $LOGDIR/ursula-4.txt

tests/conftest.py

@@ -21,6 +21,7 @@ from nucypher.blockchain.eth.networks import NetworksInventory
 from nucypher.characters.control.emitters import WebEmitter
 from nucypher.cli.config import GroupGeneralConfig
 from nucypher.crypto.powers import TransactingPower
+from nucypher.network.sensors import AvailabilitySensor
 from nucypher.utilities.logging import GlobalLoggerSettings
 from nucypher.utilities.sandbox.constants import INSECURE_DEVELOPMENT_PASSWORD, TEMPORARY_DOMAIN
 
@@ -34,6 +35,9 @@ TransactingPower.lock_account = lambda *a, **k: True
 # Disable any hardcoded preferred teachers during tests.
 TEACHER_NODES = dict()
 
+# Prevent halting the reactor via health checks during tests
+AvailabilitySensor._halt_reactor = lambda *a, **kw: True
+
 ##########################################