mirror of https://github.com/nucypher/nucypher.git
First take of NodeStorages with ABC
Kieran Prasch
parent
602443b959
commit
ffc96dcc09
|
|
@ -1,11 +0,0 @@
|
|||
from nucypher.config.parsers import parse_blockchain_config
|
||||
|
||||
|
||||
class BlockchainConfiguration:
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def from_config_file(cls, filepath: str):
|
||||
parse_blockchain_config(filepath=filepath)
|
||||
|
|
@ -0,0 +1,243 @@
|
|||
import os
|
||||
from abc import abstractmethod, ABC
|
||||
from logging import getLogger
|
||||
|
||||
import boto3 as boto3
|
||||
from typing import Set, Callable
|
||||
|
||||
from nucypher.config.constants import DEFAULT_CONFIG_ROOT
|
||||
|
||||
|
||||
class NodeStorage(ABC):
|
||||
|
||||
_name = NotImplemented
|
||||
_TYPE_LABEL = 'storage_type'
|
||||
|
||||
class NodeStorageError(Exception):
|
||||
pass
|
||||
|
||||
class NoNodeMetadataFound(NodeStorageError):
|
||||
pass
|
||||
|
||||
def __init__(self,
|
||||
serializer: Callable,
|
||||
deserializer: Callable,
|
||||
federated_only: bool, # TODO
|
||||
) -> None:
|
||||
|
||||
self.log = getLogger(self.__class__.__name__)
|
||||
self.serializer = serializer
|
||||
self.deserializer = deserializer
|
||||
self.federated_only = federated_only
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(checksum_address=item, federated_only=self.federated_only)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.save(node=value)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.remove(checksum_address=key)
|
||||
|
||||
def __iter__(self):
|
||||
return self.all(federated_only=self.federated_only)
|
||||
|
||||
@abstractmethod
|
||||
def all(self, federated_only: bool) -> set:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get(self, checksum_address: str, federated_only: bool):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def save(self, node):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def remove(self, checksum_address: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def payload(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def from_payload(self, data: str, *args, **kwargs) -> 'NodeStorage':
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def initialize(self):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class InMemoryNodeStorage(NodeStorage):
|
||||
|
||||
_name = 'memory'
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.__known_nodes = dict()
|
||||
|
||||
def all(self, federated_only: bool) -> set:
|
||||
return set(self.__known_nodes.values())
|
||||
|
||||
def get(self, checksum_address: str, federated_only: bool):
|
||||
return self.__known_nodes[checksum_address]
|
||||
|
||||
def save(self, node):
|
||||
self.__known_nodes[node.checksum_public_address] = node
|
||||
return True
|
||||
|
||||
def remove(self, checksum_address: str) -> bool:
|
||||
del self.__known_nodes[checksum_address]
|
||||
return True
|
||||
|
||||
def payload(self) -> dict:
|
||||
payload = {self._TYPE_LABEL: self._name}
|
||||
return payload
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: str, *args, **kwargs) -> 'InMemoryNodeStorage':
|
||||
if payload[cls._TYPE_LABEL] != cls._name:
|
||||
raise cls.NodeStorageError
|
||||
return cls(*args, **kwargs)
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.__known_nodes = dict()
|
||||
|
||||
|
||||
class FileBasedNodeStorage(NodeStorage):
|
||||
|
||||
_name = 'local'
|
||||
__FILENAME_TEMPLATE = '{}.node'
|
||||
__DEFAULT_DIR = os.path.join(DEFAULT_CONFIG_ROOT, 'known_nodes', 'metadata')
|
||||
|
||||
class NoNodeMetadataFound(FileNotFoundError, NodeStorage.NoNodeMetadataFound):
|
||||
pass
|
||||
|
||||
def __init__(self,
|
||||
known_metadata_dir: str = __DEFAULT_DIR,
|
||||
*args, **kwargs
|
||||
) -> None:
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
self.log = getLogger(self.__class__.__name__)
|
||||
self.known_metadata_dir = known_metadata_dir
|
||||
|
||||
def __generate_filepath(self, checksum_address: str) -> str:
|
||||
metadata_path = os.path.join(self.known_metadata_dir, self.__FILENAME_TEMPLATE.format(checksum_address))
|
||||
return metadata_path
|
||||
|
||||
def __read(self, filepath: str, federated_only: bool):
|
||||
from nucypher.characters.lawful import Ursula
|
||||
with open(filepath, "r") as seed_file:
|
||||
seed_file.seek(0)
|
||||
node_bytes = self.deserializer(seed_file.read())
|
||||
node = Ursula.from_bytes(node_bytes, federated_only=federated_only)
|
||||
return node
|
||||
|
||||
def __write(self, filepath: str, node):
|
||||
with open(filepath, "w") as f:
|
||||
f.write(self.serializer(node).hex())
|
||||
self.log.info("Wrote new node metadata to filesystem {}".format(filepath))
|
||||
return filepath
|
||||
|
||||
def all(self, federated_only: bool) -> set:
|
||||
metadata_paths = sorted(os.listdir(self.known_metadata_dir), key=os.path.getctime)
|
||||
self.log.info("Found {} known node metadata files at {}".format(len(metadata_paths), self.known_metadata_dir))
|
||||
known_nodes = set()
|
||||
for metadata_path in metadata_paths:
|
||||
node = self.__read(filepath=metadata_path, federated_only=federated_only) # TODO: 466
|
||||
known_nodes.add(node)
|
||||
return known_nodes
|
||||
|
||||
def get(self, checksum_address: str, federated_only: bool):
|
||||
metadata_path = self.__generate_filepath(checksum_address=checksum_address)
|
||||
node = self.__read(filepath=metadata_path, federated_only=federated_only) # TODO: 466
|
||||
return node
|
||||
|
||||
def save(self, node):
|
||||
try:
|
||||
filepath = self.__generate_filepath(checksum_address=node.checksum_public_address)
|
||||
except AttributeError:
|
||||
raise AttributeError("{} does not have a rest_interface attached".format(self)) # TODO.. eh?
|
||||
self.__write(filepath=filepath, node=node)
|
||||
|
||||
def remove(self, checksum_address: str):
|
||||
filepath = self.__generate_filepath(checksum_address=checksum_address)
|
||||
self.log.debug("Delted {} from the filesystem".format(checksum_address))
|
||||
return os.remove(filepath)
|
||||
|
||||
def payload(self) -> str:
|
||||
payload = {
|
||||
'storage_type': self._name,
|
||||
'known_metadata_dir': self.known_metadata_dir
|
||||
}
|
||||
return payload
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: str, *args, **kwargs) -> 'FileBasedNodeStorage':
|
||||
storage_type = payload[cls._TYPE_LABEL]
|
||||
if not storage_type == cls._name:
|
||||
raise cls.NodeStorageError("Wrong storage type. got {}".format(storage_type))
|
||||
return cls(known_metadata_dir=payload['known_metadata_dir'], *args, **kwargs)
|
||||
|
||||
def initialize(self):
|
||||
try:
|
||||
os.mkdir(self.known_metadata_dir, mode=0o755) # known_metadata
|
||||
except FileExistsError:
|
||||
message = "There are pre-existing metadata files at {}".format(self.known_metadata_dir)
|
||||
raise self.NodeStorageError(message)
|
||||
except FileNotFoundError:
|
||||
raise self.NodeStorageError("There is no existing configuration at {}".format(self.known_metadata_dir))
|
||||
|
||||
|
||||
class S3NodeStorage(NodeStorage):
|
||||
def __init__(self,
|
||||
bucket_name: str,
|
||||
*args, **kwargs) -> None:
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
self.__bucket_name = bucket_name
|
||||
self.__s3client = boto3.client('s3')
|
||||
self.__s3resource = boto3.resource('s3')
|
||||
self.bucket = self.__s3resource.Bucket(bucket_name)
|
||||
|
||||
def generate_presigned_url(self, checksum_address: str) -> str:
|
||||
payload = {'Bucket': self.__bucket_name, 'Key': checksum_address}
|
||||
url = self.__s3client.generate_presigned_url('get_object', payload)
|
||||
return url
|
||||
|
||||
def all(self, federated_only: bool) -> set:
|
||||
raise NotImplementedError # TODO
|
||||
|
||||
def get(self, checksum_address: str, federated_only: bool):
|
||||
node_obj = self.bucket.Object(checksum_address)
|
||||
node = self.deserializer(node_obj)
|
||||
return node
|
||||
|
||||
def save(self, node):
|
||||
self.__s3client.put_object(Bucket=self.__bucket_name,
|
||||
Key=node.checksum_public_address,
|
||||
Body=self.serializer(node))
|
||||
|
||||
def remove(self, checksum_address: str) -> bool:
|
||||
_node_obj = self.get(checksum_address=checksum_address, federated_only=self.federated_only)
|
||||
return _node_obj()
|
||||
|
||||
def payload(self) -> str:
|
||||
payload = {
|
||||
self._TYPE_LABEL: self._name,
|
||||
'bucket_name': self.__bucket_name
|
||||
}
|
||||
return payload
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: str, *args, **kwargs):
|
||||
return cls(bucket_name=payload['bucket_name'], *args, **kwargs)
|
||||
|
||||
def initialize(self):
|
||||
return self.__s3client.create_bucket(Bucket=self.__bucket_name)
|
||||
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
import configparser
|
||||
import os
|
||||
|
||||
from typing import Union, Tuple
|
||||
|
||||
from nucypher.config.keyring import NucypherKeyring
|
||||
from nucypher.config.node import NodeConfiguration
|
||||
|
||||
|
||||
def generate_local_wallet(keyring_root:str, passphrase: str) -> NucypherKeyring:
|
||||
keyring = NucypherKeyring.generate(passphrase=passphrase,
|
||||
keyring_root=keyring_root,
|
||||
encrypting=False,
|
||||
wallet=True)
|
||||
return keyring
|
||||
|
||||
|
||||
def generate_account(w3, passphrase: str) -> NucypherKeyring:
|
||||
address = w3.personal.newAccount(passphrase)
|
||||
return address
|
||||
|
||||
|
||||
def check_config_permissions() -> bool:
|
||||
rules = (
|
||||
(os.name == 'nt' or os.getuid() != 0, 'Cannot run as root user.'),
|
||||
)
|
||||
|
||||
for rule, failure_reason in rules:
|
||||
if rule is not True:
|
||||
raise Exception(failure_reason)
|
||||
return True
|
||||
|
||||
|
||||
def validate_configuration_file(config=None,
|
||||
filepath: str = NodeConfiguration.DEFAULT_CONFIG_FILE_LOCATION,
|
||||
raise_on_failure: bool=False) -> Union[bool, Tuple[bool, tuple]]:
|
||||
|
||||
if config is None:
|
||||
config = configparser.ConfigParser()
|
||||
config.read(filepath)
|
||||
|
||||
if not config.sections():
|
||||
|
||||
raise NodeConfiguration.InvalidConfiguration("Empty configuration file")
|
||||
|
||||
required_sections = ("nucypher", "blockchain")
|
||||
|
||||
missing_sections = list()
|
||||
|
||||
try:
|
||||
operating_mode = config["nucypher"]["mode"]
|
||||
except KeyError:
|
||||
raise NodeConfiguration.ConfigurationError("No operating mode configured")
|
||||
else:
|
||||
modes = ('federated', 'tester', 'decentralized', 'centralized')
|
||||
if operating_mode not in modes:
|
||||
missing_sections.append("mode")
|
||||
if raise_on_failure is True:
|
||||
raise NodeConfiguration.ConfigurationError("Invalid nucypher operating mode '{}'. Specify {}".format(operating_mode, modes))
|
||||
|
||||
for section in required_sections:
|
||||
if section not in config.sections():
|
||||
missing_sections.append(section)
|
||||
if raise_on_failure is True:
|
||||
raise NodeConfiguration.ConfigurationError("Invalid config file: missing section '{}'".format(section))
|
||||
|
||||
if len(missing_sections) > 0:
|
||||
result = False, tuple(missing_sections)
|
||||
else:
|
||||
result = True, tuple()
|
||||
|
||||
return result
|
||||
|
|
@ -156,7 +156,7 @@ class ProxyRESTRoutes:
|
|||
self._suspicious_activity_tracker['vladimirs'].append(node) # TODO: Maybe also record the bytes representation separately to disk?
|
||||
except Exception as e:
|
||||
self.log.critical(str(e))
|
||||
raise
|
||||
raise # TODO
|
||||
else:
|
||||
self.log.info("Previously unknown node: {}".format(node.checksum_public_address))
|
||||
self._node_recorder(node)
|
||||
|
|
|
|||
Loading…
Reference in New Issue