From ffc96dcc095f9fd5f9a8a74664b5195ffff26fe8 Mon Sep 17 00:00:00 2001 From: Kieran Prasch Date: Tue, 9 Oct 2018 12:40:49 -0700 Subject: [PATCH] First take of NodeStorages with ABC --- nucypher/config/blockchain.py | 11 -- nucypher/config/storages.py | 243 ++++++++++++++++++++++++++++++++++ nucypher/config/utils.py | 72 ---------- nucypher/network/server.py | 2 +- 4 files changed, 244 insertions(+), 84 deletions(-) delete mode 100644 nucypher/config/blockchain.py create mode 100644 nucypher/config/storages.py delete mode 100644 nucypher/config/utils.py diff --git a/nucypher/config/blockchain.py b/nucypher/config/blockchain.py deleted file mode 100644 index bd91885c0..000000000 --- a/nucypher/config/blockchain.py +++ /dev/null @@ -1,11 +0,0 @@ -from nucypher.config.parsers import parse_blockchain_config - - -class BlockchainConfiguration: - - def __init__(self): - pass - - @classmethod - def from_config_file(cls, filepath: str): - parse_blockchain_config(filepath=filepath) diff --git a/nucypher/config/storages.py b/nucypher/config/storages.py new file mode 100644 index 000000000..5ba04cfc6 --- /dev/null +++ b/nucypher/config/storages.py @@ -0,0 +1,243 @@ +import os +from abc import abstractmethod, ABC +from logging import getLogger + +import boto3 as boto3 +from typing import Set, Callable + +from nucypher.config.constants import DEFAULT_CONFIG_ROOT + + +class NodeStorage(ABC): + + _name = NotImplemented + _TYPE_LABEL = 'storage_type' + + class NodeStorageError(Exception): + pass + + class NoNodeMetadataFound(NodeStorageError): + pass + + def __init__(self, + serializer: Callable, + deserializer: Callable, + federated_only: bool, # TODO + ) -> None: + + self.log = getLogger(self.__class__.__name__) + self.serializer = serializer + self.deserializer = deserializer + self.federated_only = federated_only + + def __getitem__(self, item): + return self.get(checksum_address=item, federated_only=self.federated_only) + + def __setitem__(self, key, value): + return self.save(node=value) + + def __delitem__(self, key): + self.remove(checksum_address=key) + + def __iter__(self): + return self.all(federated_only=self.federated_only) + + @abstractmethod + def all(self, federated_only: bool) -> set: + raise NotImplementedError + + @abstractmethod + def get(self, checksum_address: str, federated_only: bool): + raise NotImplementedError + + @abstractmethod + def save(self, node): + raise NotImplementedError + + @abstractmethod + def remove(self, checksum_address: str) -> bool: + raise NotImplementedError + + @abstractmethod + def payload(self) -> str: + raise NotImplementedError + + @classmethod + @abstractmethod + def from_payload(self, data: str, *args, **kwargs) -> 'NodeStorage': + raise NotImplementedError + + @abstractmethod + def initialize(self): + raise NotImplementedError + + +class InMemoryNodeStorage(NodeStorage): + + _name = 'memory' + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.__known_nodes = dict() + + def all(self, federated_only: bool) -> set: + return set(self.__known_nodes.values()) + + def get(self, checksum_address: str, federated_only: bool): + return self.__known_nodes[checksum_address] + + def save(self, node): + self.__known_nodes[node.checksum_public_address] = node + return True + + def remove(self, checksum_address: str) -> bool: + del self.__known_nodes[checksum_address] + return True + + def payload(self) -> dict: + payload = {self._TYPE_LABEL: self._name} + return payload + + @classmethod + def from_payload(cls, payload: str, *args, **kwargs) -> 'InMemoryNodeStorage': + if payload[cls._TYPE_LABEL] != cls._name: + raise cls.NodeStorageError + return cls(*args, **kwargs) + + def initialize(self) -> None: + self.__known_nodes = dict() + + +class FileBasedNodeStorage(NodeStorage): + + _name = 'local' + __FILENAME_TEMPLATE = '{}.node' + __DEFAULT_DIR = os.path.join(DEFAULT_CONFIG_ROOT, 'known_nodes', 'metadata') + + class NoNodeMetadataFound(FileNotFoundError, NodeStorage.NoNodeMetadataFound): + pass + + def __init__(self, + known_metadata_dir: str = __DEFAULT_DIR, + *args, **kwargs + ) -> None: + + super().__init__(*args, **kwargs) + self.log = getLogger(self.__class__.__name__) + self.known_metadata_dir = known_metadata_dir + + def __generate_filepath(self, checksum_address: str) -> str: + metadata_path = os.path.join(self.known_metadata_dir, self.__FILENAME_TEMPLATE.format(checksum_address)) + return metadata_path + + def __read(self, filepath: str, federated_only: bool): + from nucypher.characters.lawful import Ursula + with open(filepath, "r") as seed_file: + seed_file.seek(0) + node_bytes = self.deserializer(seed_file.read()) + node = Ursula.from_bytes(node_bytes, federated_only=federated_only) + return node + + def __write(self, filepath: str, node): + with open(filepath, "w") as f: + f.write(self.serializer(node).hex()) + self.log.info("Wrote new node metadata to filesystem {}".format(filepath)) + return filepath + + def all(self, federated_only: bool) -> set: + metadata_paths = sorted(os.listdir(self.known_metadata_dir), key=os.path.getctime) + self.log.info("Found {} known node metadata files at {}".format(len(metadata_paths), self.known_metadata_dir)) + known_nodes = set() + for metadata_path in metadata_paths: + node = self.__read(filepath=metadata_path, federated_only=federated_only) # TODO: 466 + known_nodes.add(node) + return known_nodes + + def get(self, checksum_address: str, federated_only: bool): + metadata_path = self.__generate_filepath(checksum_address=checksum_address) + node = self.__read(filepath=metadata_path, federated_only=federated_only) # TODO: 466 + return node + + def save(self, node): + try: + filepath = self.__generate_filepath(checksum_address=node.checksum_public_address) + except AttributeError: + raise AttributeError("{} does not have a rest_interface attached".format(self)) # TODO.. eh? + self.__write(filepath=filepath, node=node) + + def remove(self, checksum_address: str): + filepath = self.__generate_filepath(checksum_address=checksum_address) + self.log.debug("Delted {} from the filesystem".format(checksum_address)) + return os.remove(filepath) + + def payload(self) -> str: + payload = { + 'storage_type': self._name, + 'known_metadata_dir': self.known_metadata_dir + } + return payload + + @classmethod + def from_payload(cls, payload: str, *args, **kwargs) -> 'FileBasedNodeStorage': + storage_type = payload[cls._TYPE_LABEL] + if not storage_type == cls._name: + raise cls.NodeStorageError("Wrong storage type. got {}".format(storage_type)) + return cls(known_metadata_dir=payload['known_metadata_dir'], *args, **kwargs) + + def initialize(self): + try: + os.mkdir(self.known_metadata_dir, mode=0o755) # known_metadata + except FileExistsError: + message = "There are pre-existing metadata files at {}".format(self.known_metadata_dir) + raise self.NodeStorageError(message) + except FileNotFoundError: + raise self.NodeStorageError("There is no existing configuration at {}".format(self.known_metadata_dir)) + + +class S3NodeStorage(NodeStorage): + def __init__(self, + bucket_name: str, + *args, **kwargs) -> None: + + super().__init__(*args, **kwargs) + self.__bucket_name = bucket_name + self.__s3client = boto3.client('s3') + self.__s3resource = boto3.resource('s3') + self.bucket = self.__s3resource.Bucket(bucket_name) + + def generate_presigned_url(self, checksum_address: str) -> str: + payload = {'Bucket': self.__bucket_name, 'Key': checksum_address} + url = self.__s3client.generate_presigned_url('get_object', payload) + return url + + def all(self, federated_only: bool) -> set: + raise NotImplementedError # TODO + + def get(self, checksum_address: str, federated_only: bool): + node_obj = self.bucket.Object(checksum_address) + node = self.deserializer(node_obj) + return node + + def save(self, node): + self.__s3client.put_object(Bucket=self.__bucket_name, + Key=node.checksum_public_address, + Body=self.serializer(node)) + + def remove(self, checksum_address: str) -> bool: + _node_obj = self.get(checksum_address=checksum_address, federated_only=self.federated_only) + return _node_obj() + + def payload(self) -> str: + payload = { + self._TYPE_LABEL: self._name, + 'bucket_name': self.__bucket_name + } + return payload + + @classmethod + def from_payload(cls, payload: str, *args, **kwargs): + return cls(bucket_name=payload['bucket_name'], *args, **kwargs) + + def initialize(self): + return self.__s3client.create_bucket(Bucket=self.__bucket_name) + diff --git a/nucypher/config/utils.py b/nucypher/config/utils.py deleted file mode 100644 index fee2a9321..000000000 --- a/nucypher/config/utils.py +++ /dev/null @@ -1,72 +0,0 @@ -import configparser -import os - -from typing import Union, Tuple - -from nucypher.config.keyring import NucypherKeyring -from nucypher.config.node import NodeConfiguration - - -def generate_local_wallet(keyring_root:str, passphrase: str) -> NucypherKeyring: - keyring = NucypherKeyring.generate(passphrase=passphrase, - keyring_root=keyring_root, - encrypting=False, - wallet=True) - return keyring - - -def generate_account(w3, passphrase: str) -> NucypherKeyring: - address = w3.personal.newAccount(passphrase) - return address - - -def check_config_permissions() -> bool: - rules = ( - (os.name == 'nt' or os.getuid() != 0, 'Cannot run as root user.'), - ) - - for rule, failure_reason in rules: - if rule is not True: - raise Exception(failure_reason) - return True - - -def validate_configuration_file(config=None, - filepath: str = NodeConfiguration.DEFAULT_CONFIG_FILE_LOCATION, - raise_on_failure: bool=False) -> Union[bool, Tuple[bool, tuple]]: - - if config is None: - config = configparser.ConfigParser() - config.read(filepath) - - if not config.sections(): - - raise NodeConfiguration.InvalidConfiguration("Empty configuration file") - - required_sections = ("nucypher", "blockchain") - - missing_sections = list() - - try: - operating_mode = config["nucypher"]["mode"] - except KeyError: - raise NodeConfiguration.ConfigurationError("No operating mode configured") - else: - modes = ('federated', 'tester', 'decentralized', 'centralized') - if operating_mode not in modes: - missing_sections.append("mode") - if raise_on_failure is True: - raise NodeConfiguration.ConfigurationError("Invalid nucypher operating mode '{}'. Specify {}".format(operating_mode, modes)) - - for section in required_sections: - if section not in config.sections(): - missing_sections.append(section) - if raise_on_failure is True: - raise NodeConfiguration.ConfigurationError("Invalid config file: missing section '{}'".format(section)) - - if len(missing_sections) > 0: - result = False, tuple(missing_sections) - else: - result = True, tuple() - - return result diff --git a/nucypher/network/server.py b/nucypher/network/server.py index 689067024..778aabdd6 100644 --- a/nucypher/network/server.py +++ b/nucypher/network/server.py @@ -156,7 +156,7 @@ class ProxyRESTRoutes: self._suspicious_activity_tracker['vladimirs'].append(node) # TODO: Maybe also record the bytes representation separately to disk? except Exception as e: self.log.critical(str(e)) - raise + raise # TODO else: self.log.info("Previously unknown node: {}".format(node.checksum_public_address)) self._node_recorder(node)