Refines handling of unresponsive nodes at ping time.

pull/1462/head
Kieran R. Prasch 2019-12-19 15:08:41 -08:00 committed by Kieran Prasch
parent 64786b572d
commit 05b6dcf26c
No known key found for this signature in database
GPG Key ID: 199AB839D4125A62
3 changed files with 14 additions and 10 deletions

View File

@ -122,7 +122,6 @@ def paint_node_status(emitter, ursula, start_time):
'{}'.format(ursula),
'Uptime .............. {}'.format(maya.now() - start_time),
'Start Time .......... {}'.format(start_time.slang_time()),
'Availability Score .. {}'.format(ursula._availability_sensor.score),
'Fleet State.......... {}'.format(fleet_state),
'Learning Status ..... {}'.format(learning_status),
'Learning Round ...... Round #{}'.format(ursula._learning_round),
@ -138,6 +137,10 @@ def paint_node_status(emitter, ursula, start_time):
current_period = f'Current Period ...... {ursula.staking_agent.get_current_period()}'
stats.extend([current_period, worker_address])
if ursula._availability_sensor:
score = 'Availability Score .. {}'.format(ursula._availability_sensor.score),
stats.append(score)
emitter.echo('\n' + '\n'.join(stats) + '\n')

View File

@ -167,8 +167,8 @@ class RestMiddleware:
if current_attempt == retry_attempts:
message = f"No Response from seednode {host}:{port} after {retry_attempts} attempts"
self.log.info(message)
raise RuntimeError("No response from {}:{}".format(host, port))
self.log.info("No Response from seednode {}. Retrying in {} seconds...".format(host, retry_rate))
raise ConnectionRefusedError("No response from {}:{}".format(host, port))
self.log.info(f"No Response from seednode {host}:{port}. Retrying in {retry_rate} seconds...")
time.sleep(retry_rate)
return self.get_certificate(host, port, timeout, retry_attempts, retry_rate, current_attempt + 1)

View File

@ -13,8 +13,8 @@ class AvailabilitySensor:
FAST_INTERVAL = 5 # Seconds
SLOW_INTERVAL = 60 * 5
SEEDING_DURATION = 60 * 2
MAXIMUM_ALONE_TIME = 10
SEEDING_DURATION = 60
MAXIMUM_ALONE_TIME = 120
MAXIMUM_SCORE = 10.0 # Score
SAMPLE_SIZE = 1 # Ursulas
@ -49,10 +49,10 @@ class AvailabilitySensor:
self.__task = LoopingCall(self.maintain)
def mild_warning(self) -> None:
self.log.info(f'[UNREACHABLE NOTICE] {self._ursula.rest_url} was recently reported as unreachable.')
self.log.info(f'[UNREACHABLE NOTICE] This node was recently reported as unreachable.')
def medium_warning(self) -> None:
self.log.warn(f'[UNREACHABLE CAUTION] {self._ursula.rest_url} is reporting as unreachable.'
self.log.warn(f'[UNREACHABLE CAUTION] This node is reporting as unreachable.'
f'Please check your network and firewall configuration.')
def severe_warning(self) -> None:
@ -158,12 +158,13 @@ class AvailabilitySensor:
# Fetch and store teacher certificate
responding_ursula_address, responding_ursula_port = tuple(ursula.rest_interface)
certificate = self._ursula.network_middleware.get_certificate(host=responding_ursula_address,
port=responding_ursula_port)
certificate_filepath = self._ursula.node_storage.store_node_certificate(certificate=certificate)
# Request status check
try:
certificate = self._ursula.network_middleware.get_certificate(host=responding_ursula_address,
port=responding_ursula_port)
certificate_filepath = self._ursula.node_storage.store_node_certificate(certificate=certificate)
response = self._ursula.network_middleware.check_rest_availability(requesting_ursula=self._ursula,
responding_ursula=ursula,
certificate_filepath=certificate_filepath)