First take of NodeStorages with ABC

nucypher/config/blockchain.py

@@ -1,11 +0,0 @@
-from nucypher.config.parsers import parse_blockchain_config
-
-
-class BlockchainConfiguration:
-
-    def __init__(self):
-        pass
-
-    @classmethod
-    def from_config_file(cls, filepath: str):
-        parse_blockchain_config(filepath=filepath)

nucypher/config/storages.py

@@ -0,0 +1,243 @@
+import os
+from abc import abstractmethod, ABC
+from logging import getLogger
+
+import boto3 as boto3
+from typing import Set, Callable
+
+from nucypher.config.constants import DEFAULT_CONFIG_ROOT
+
+
+class NodeStorage(ABC):
+
+    _name = NotImplemented
+    _TYPE_LABEL = 'storage_type'
+
+    class NodeStorageError(Exception):
+        pass
+
+    class NoNodeMetadataFound(NodeStorageError):
+        pass
+
+    def __init__(self,
+                 serializer: Callable,
+                 deserializer: Callable,
+                 federated_only: bool,  # TODO
+                 ) -> None:
+
+        self.log = getLogger(self.__class__.__name__)
+        self.serializer = serializer
+        self.deserializer = deserializer
+        self.federated_only = federated_only
+
+    def __getitem__(self, item):
+        return self.get(checksum_address=item, federated_only=self.federated_only)
+
+    def __setitem__(self, key, value):
+        return self.save(node=value)
+
+    def __delitem__(self, key):
+        self.remove(checksum_address=key)
+
+    def __iter__(self):
+        return self.all(federated_only=self.federated_only)
+
+    @abstractmethod
+    def all(self, federated_only: bool) -> set:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get(self, checksum_address: str, federated_only: bool):
+        raise NotImplementedError
+
+    @abstractmethod
+    def save(self, node):
+        raise NotImplementedError
+
+    @abstractmethod
+    def remove(self, checksum_address: str) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def payload(self) -> str:
+        raise NotImplementedError
+
+    @classmethod
+    @abstractmethod
+    def from_payload(self, data: str, *args, **kwargs) -> 'NodeStorage':
+        raise NotImplementedError
+
+    @abstractmethod
+    def initialize(self):
+        raise NotImplementedError
+
+
+class InMemoryNodeStorage(NodeStorage):
+
+    _name = 'memory'
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.__known_nodes = dict()
+
+    def all(self, federated_only: bool) -> set:
+        return set(self.__known_nodes.values())
+
+    def get(self, checksum_address: str, federated_only: bool):
+        return self.__known_nodes[checksum_address]
+
+    def save(self, node):
+        self.__known_nodes[node.checksum_public_address] = node
+        return True
+
+    def remove(self, checksum_address: str) -> bool:
+        del self.__known_nodes[checksum_address]
+        return True
+
+    def payload(self) -> dict:
+        payload = {self._TYPE_LABEL: self._name}
+        return payload
+
+    @classmethod
+    def from_payload(cls, payload: str, *args, **kwargs) -> 'InMemoryNodeStorage':
+        if payload[cls._TYPE_LABEL] != cls._name:
+            raise cls.NodeStorageError
+        return cls(*args, **kwargs)
+
+    def initialize(self) -> None:
+        self.__known_nodes = dict()
+
+
+class FileBasedNodeStorage(NodeStorage):
+
+    _name = 'local'
+    __FILENAME_TEMPLATE = '{}.node'
+    __DEFAULT_DIR = os.path.join(DEFAULT_CONFIG_ROOT, 'known_nodes', 'metadata')
+
+    class NoNodeMetadataFound(FileNotFoundError, NodeStorage.NoNodeMetadataFound):
+        pass
+
+    def __init__(self,
+                 known_metadata_dir: str = __DEFAULT_DIR,
+                 *args, **kwargs
+                 ) -> None:
+
+        super().__init__(*args, **kwargs)
+        self.log = getLogger(self.__class__.__name__)
+        self.known_metadata_dir = known_metadata_dir
+
+    def __generate_filepath(self, checksum_address: str) -> str:
+        metadata_path = os.path.join(self.known_metadata_dir, self.__FILENAME_TEMPLATE.format(checksum_address))
+        return metadata_path
+
+    def __read(self, filepath: str, federated_only: bool):
+        from nucypher.characters.lawful import Ursula
+        with open(filepath, "r") as seed_file:
+            seed_file.seek(0)
+            node_bytes = self.deserializer(seed_file.read())
+            node = Ursula.from_bytes(node_bytes, federated_only=federated_only)
+        return node
+
+    def __write(self, filepath: str, node):
+        with open(filepath, "w") as f:
+            f.write(self.serializer(node).hex())
+        self.log.info("Wrote new node metadata to filesystem {}".format(filepath))
+        return filepath
+
+    def all(self, federated_only: bool) -> set:
+        metadata_paths = sorted(os.listdir(self.known_metadata_dir), key=os.path.getctime)
+        self.log.info("Found {} known node metadata files at {}".format(len(metadata_paths), self.known_metadata_dir))
+        known_nodes = set()
+        for metadata_path in metadata_paths:
+            node = self.__read(filepath=metadata_path, federated_only=federated_only)   # TODO: 466
+            known_nodes.add(node)
+        return known_nodes
+
+    def get(self, checksum_address: str, federated_only: bool):
+        metadata_path = self.__generate_filepath(checksum_address=checksum_address)
+        node = self.__read(filepath=metadata_path, federated_only=federated_only)   # TODO: 466
+        return node
+
+    def save(self, node):
+        try:
+            filepath = self.__generate_filepath(checksum_address=node.checksum_public_address)
+        except AttributeError:
+            raise AttributeError("{} does not have a rest_interface attached".format(self))  # TODO.. eh?
+        self.__write(filepath=filepath, node=node)
+
+    def remove(self, checksum_address: str):
+        filepath = self.__generate_filepath(checksum_address=checksum_address)
+        self.log.debug("Delted {} from the filesystem".format(checksum_address))
+        return os.remove(filepath)
+
+    def payload(self) -> str:
+        payload = {
+            'storage_type': self._name,
+            'known_metadata_dir': self.known_metadata_dir
+        }
+        return payload
+
+    @classmethod
+    def from_payload(cls, payload: str, *args, **kwargs) -> 'FileBasedNodeStorage':
+        storage_type = payload[cls._TYPE_LABEL]
+        if not storage_type == cls._name:
+            raise cls.NodeStorageError("Wrong storage type. got {}".format(storage_type))
+        return cls(known_metadata_dir=payload['known_metadata_dir'], *args, **kwargs)
+
+    def initialize(self):
+        try:
+            os.mkdir(self.known_metadata_dir, mode=0o755)  # known_metadata
+        except FileExistsError:
+            message = "There are pre-existing metadata files at {}".format(self.known_metadata_dir)
+            raise self.NodeStorageError(message)
+        except FileNotFoundError:
+            raise self.NodeStorageError("There is no existing configuration at {}".format(self.known_metadata_dir))
+
+
+class S3NodeStorage(NodeStorage):
+    def __init__(self,
+                 bucket_name: str,
+                 *args, **kwargs) -> None:
+
+        super().__init__(*args, **kwargs)
+        self.__bucket_name = bucket_name
+        self.__s3client = boto3.client('s3')
+        self.__s3resource = boto3.resource('s3')
+        self.bucket = self.__s3resource.Bucket(bucket_name)
+
+    def generate_presigned_url(self, checksum_address: str) -> str:
+        payload = {'Bucket': self.__bucket_name, 'Key': checksum_address}
+        url = self.__s3client.generate_presigned_url('get_object', payload)
+        return url
+
+    def all(self, federated_only: bool) -> set:
+        raise NotImplementedError  # TODO
+
+    def get(self, checksum_address: str, federated_only: bool):
+        node_obj = self.bucket.Object(checksum_address)
+        node = self.deserializer(node_obj)
+        return node
+
+    def save(self, node):
+        self.__s3client.put_object(Bucket=self.__bucket_name,
+                                   Key=node.checksum_public_address,
+                                   Body=self.serializer(node))
+
+    def remove(self, checksum_address: str) -> bool:
+        _node_obj = self.get(checksum_address=checksum_address, federated_only=self.federated_only)
+        return _node_obj()
+
+    def payload(self) -> str:
+        payload = {
+            self._TYPE_LABEL: self._name,
+            'bucket_name': self.__bucket_name
+        }
+        return payload
+
+    @classmethod
+    def from_payload(cls, payload: str, *args, **kwargs):
+        return cls(bucket_name=payload['bucket_name'], *args, **kwargs)
+
+    def initialize(self):
+        return self.__s3client.create_bucket(Bucket=self.__bucket_name)
+

nucypher/config/utils.py

@@ -1,72 +0,0 @@
-import configparser
-import os
-
-from typing import Union, Tuple
-
-from nucypher.config.keyring import NucypherKeyring
-from nucypher.config.node import NodeConfiguration
-
-
-def generate_local_wallet(keyring_root:str, passphrase: str) -> NucypherKeyring:
-    keyring = NucypherKeyring.generate(passphrase=passphrase,
-                                       keyring_root=keyring_root,
-                                       encrypting=False,
-                                       wallet=True)
-    return keyring
-
-
-def generate_account(w3, passphrase: str) -> NucypherKeyring:
-    address = w3.personal.newAccount(passphrase)
-    return address
-
-
-def check_config_permissions() -> bool:
-    rules = (
-        (os.name == 'nt' or os.getuid() != 0, 'Cannot run as root user.'),
-    )
-
-    for rule, failure_reason in rules:
-        if rule is not True:
-            raise Exception(failure_reason)
-    return True
-
-
-def validate_configuration_file(config=None,
-                                filepath: str = NodeConfiguration.DEFAULT_CONFIG_FILE_LOCATION,
-                                raise_on_failure: bool=False) -> Union[bool, Tuple[bool, tuple]]:
-
-    if config is None:
-        config = configparser.ConfigParser()
-        config.read(filepath)
-
-    if not config.sections():
-
-        raise NodeConfiguration.InvalidConfiguration("Empty configuration file")
-
-    required_sections = ("nucypher", "blockchain")
-
-    missing_sections = list()
-
-    try:
-        operating_mode = config["nucypher"]["mode"]
-    except KeyError:
-        raise NodeConfiguration.ConfigurationError("No operating mode configured")
-    else:
-        modes = ('federated', 'tester', 'decentralized', 'centralized')
-        if operating_mode not in modes:
-            missing_sections.append("mode")
-            if raise_on_failure is True:
-                raise NodeConfiguration.ConfigurationError("Invalid nucypher operating mode '{}'. Specify {}".format(operating_mode, modes))
-
-    for section in required_sections:
-        if section not in config.sections():
-            missing_sections.append(section)
-            if raise_on_failure is True:
-                raise NodeConfiguration.ConfigurationError("Invalid config file: missing section '{}'".format(section))
-
-    if len(missing_sections) > 0:
-        result = False, tuple(missing_sections)
-    else:
-        result = True, tuple()
-
-    return result

nucypher/network/server.py

@@ -156,7 +156,7 @@ class ProxyRESTRoutes:
                     self._suspicious_activity_tracker['vladimirs'].append(node)  # TODO: Maybe also record the bytes representation separately to disk?
                 except Exception as e:
                     self.log.critical(str(e))
-                    raise
+                    raise  # TODO
                 else:
                     self.log.info("Previously unknown node: {}".format(node.checksum_public_address))
                     self._node_recorder(node)