diff --git a/docs/source/conf.py b/docs/source/conf.py index ee8593efe..ef281d98f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -253,6 +253,7 @@ def run_apidoc(_): 'scripts', Path('nucypher', 'utilities'), Path('nucypher', 'blockchain', 'eth', 'sol'), + Path('nucypher', 'blockchain', 'eth', 'economics.py'), ] for exclusion_item in exclusion_items: apidoc_command.append(f'{nucypher_module_dir / exclusion_item}') diff --git a/newsfragments/2439.misc.rst b/newsfragments/2439.misc.rst new file mode 100644 index 000000000..a7aac05e1 --- /dev/null +++ b/newsfragments/2439.misc.rst @@ -0,0 +1 @@ +Rework internal solidity compiler usage to implement "Standard JSON Compile". diff --git a/nucypher/blockchain/eth/interfaces.py b/nucypher/blockchain/eth/interfaces.py index e63421bd0..98266e727 100644 --- a/nucypher/blockchain/eth/interfaces.py +++ b/nucypher/blockchain/eth/interfaces.py @@ -18,12 +18,26 @@ along with nucypher. If not, see . import math import os import pprint +import threading import time from typing import Callable, NamedTuple, Tuple, Union, Optional +from typing import List from urllib.parse import urlparse import click import requests +from eth_tester import EthereumTester +from eth_tester.exceptions import TransactionFailed as TestTransactionFailed +from eth_typing import ChecksumAddress +from eth_utils import to_checksum_address +from hexbytes.main import HexBytes +from web3 import Web3, middleware, IPCProvider, WebsocketProvider, HTTPProvider +from web3.contract import Contract, ContractConstructor, ContractFunction +from web3.exceptions import ValidationError, TimeExhausted +from web3.middleware import geth_poa_middleware +from web3.providers import BaseProvider +from web3.types import TxReceipt + from constant_sorrow.constants import ( INSUFFICIENT_ETH, NO_BLOCKCHAIN_CONNECTION, @@ -32,36 +46,29 @@ from constant_sorrow.constants import ( READ_ONLY_INTERFACE, UNKNOWN_TX_STATUS ) -from eth_tester.exceptions import TransactionFailed as TestTransactionFailed -from eth_typing import ChecksumAddress -from eth_utils import to_checksum_address -from hexbytes.main import HexBytes -from web3 import Web3, middleware -from web3.contract import Contract, ContractConstructor, ContractFunction -from web3.exceptions import TimeExhausted, ValidationError -from web3.middleware import geth_poa_middleware -from web3.providers import BaseProvider -from web3.types import TxReceipt - from nucypher.blockchain.eth.clients import EthereumClient, POA_CHAINS, InfuraClient from nucypher.blockchain.eth.decorators import validate_checksum_address from nucypher.blockchain.eth.providers import ( - _get_auto_provider, _get_HTTP_provider, _get_IPC_provider, + _get_auto_provider, _get_mock_test_provider, _get_pyevm_test_provider, _get_test_geth_parity_provider, _get_websocket_provider ) from nucypher.blockchain.eth.registry import BaseContractRegistry -from nucypher.blockchain.eth.sol.compile import SolidityCompiler +from nucypher.blockchain.eth.sol.compile.compile import multiversion_compile +from nucypher.blockchain.eth.sol.compile.constants import SOLIDITY_SOURCE_ROOT +from nucypher.blockchain.eth.sol.compile.types import SourceBundle from nucypher.blockchain.eth.utils import get_transaction_name, prettify_eth_amount from nucypher.characters.control.emitters import JSONRPCStdoutEmitter, StdoutEmitter from nucypher.utilities.ethereum import encode_constructor_arguments from nucypher.utilities.gas_strategies import datafeed_fallback_gas_price_strategy, WEB3_GAS_STRATEGIES from nucypher.utilities.logging import GlobalLoggerSettings, Logger +Web3Providers = Union[IPCProvider, WebsocketProvider, HTTPProvider, EthereumTester] # TODO: Move to types.py + class VersionedContract(Contract): version = None @@ -79,9 +86,9 @@ class BlockchainInterface: GAS_STRATEGIES = WEB3_GAS_STRATEGIES process = NO_PROVIDER_PROCESS.bool_value(False) - Web3 = Web3 + Web3 = Web3 # TODO: This is name-shadowing the actual Web3. Is this intentional? - _contract_factory = VersionedContract + _CONTRACT_FACTORY = VersionedContract class InterfaceError(Exception): pass @@ -227,7 +234,7 @@ class BlockchainInterface: self._provider = provider self._provider_process = provider_process self.w3 = NO_BLOCKCHAIN_CONNECTION - self.client = NO_BLOCKCHAIN_CONNECTION # type: EthereumClient + self.client = NO_BLOCKCHAIN_CONNECTION self.transacting_power = READ_ONLY_INTERFACE self.is_light = light self.gas_strategy = gas_strategy or self.DEFAULT_GAS_STRATEGY @@ -602,7 +609,6 @@ class BlockchainInterface: # # Broadcast # - emitter.message(f'Broadcasting {transaction_name} Transaction ({cost} ETH @ {price_gwei} gwei)...', color='yellow') try: @@ -716,7 +722,7 @@ class BlockchainInterface: proxy_contract = self.client.w3.eth.contract(abi=proxy_abi, address=proxy_address, version=proxy_version, - ContractFactoryClass=self._contract_factory) + ContractFactoryClass=self._CONTRACT_FACTORY) # Read this dispatcher's target address from the blockchain proxy_live_target_address = proxy_contract.functions.target().call() @@ -768,7 +774,7 @@ class BlockchainInterface: unified_contract = self.client.w3.eth.contract(abi=selected_abi, address=selected_address, version=selected_version, - ContractFactoryClass=self._contract_factory) + ContractFactoryClass=self._CONTRACT_FACTORY) return unified_contract @@ -797,7 +803,15 @@ class BlockchainInterface: class BlockchainDeployerInterface(BlockchainInterface): TIMEOUT = 600 # seconds - _contract_factory = VersionedContract + _CONTRACT_FACTORY = VersionedContract + + # TODO: Make more func - use as a parameter + # Source directories to (recursively) compile + SOURCES: List[SourceBundle] = [ + SourceBundle(base_path=SOLIDITY_SOURCE_ROOT), + ] + + _raw_contract_cache = NO_COMPILATION_PERFORMED class NoDeployerAddress(RuntimeError): pass @@ -805,32 +819,15 @@ class BlockchainDeployerInterface(BlockchainInterface): class DeploymentFailed(RuntimeError): pass - def __init__(self, - compiler: SolidityCompiler = None, - ignore_solidity_check: bool = False, - dry_run: bool = False, - *args, **kwargs): - - super().__init__(*args, **kwargs) - self.dry_run = dry_run - self.compiler = compiler or SolidityCompiler(ignore_solidity_check=ignore_solidity_check) - - def connect(self): + def connect(self, compile_now: bool = True, ignore_solidity_check: bool = False) -> bool: super().connect() - self._setup_solidity(compiler=self.compiler) - return self.is_connected - - def _setup_solidity(self, compiler: SolidityCompiler = None) -> None: - if self.dry_run: - self.log.info("Dry run is active, skipping solidity compile steps.") - return - if compiler: + if compile_now: # Execute the compilation if we're recompiling # Otherwise read compiled contract data from the registry. - _raw_contract_cache = compiler.compile() - else: - _raw_contract_cache = NO_COMPILATION_PERFORMED - self._raw_contract_cache = _raw_contract_cache + check = not ignore_solidity_check + compiled_contracts = multiversion_compile(source_bundles=self.SOURCES, compiler_version_check=check) + self._raw_contract_cache = compiled_contracts + return self.is_connected @validate_checksum_address def deploy_contract(self, @@ -892,7 +889,7 @@ class BlockchainDeployerInterface(BlockchainInterface): contract = self.client.w3.eth.contract(address=address, abi=contract_factory.abi, version=contract_factory.version, - ContractFactoryClass=self._contract_factory) + ContractFactoryClass=self._CONTRACT_FACTORY) if enroll is True: registry.enroll(contract_name=contract_name, @@ -917,9 +914,9 @@ class BlockchainDeployerInterface(BlockchainInterface): return requested_version, contract_data[requested_version] except KeyError: if requested_version != 'latest' and requested_version != 'earliest': - raise self.UnknownContract('Version {} of contract {} is not a locally compiled. ' - 'Available versions: {}' - .format(requested_version, contract_name, contract_data.keys())) + available = ', '.join(contract_data.keys()) + raise self.UnknownContract(f'Version {contract_name} of contract {contract_name} is not a locally compiled. ' + f'Available versions: {available}') if len(contract_data.keys()) == 1: return next(iter(contract_data.items())) @@ -945,10 +942,10 @@ class BlockchainDeployerInterface(BlockchainInterface): """Retrieve compiled interface data from the cache and return web3 contract""" version, interface = self.find_raw_contract_data(contract_name, version) contract = self.client.w3.eth.contract(abi=interface['abi'], - bytecode=interface['bin'], + bytecode=interface['evm']['bytecode']['object'], version=version, address=address, - ContractFactoryClass=self._contract_factory) + ContractFactoryClass=self._CONTRACT_FACTORY) return contract def get_contract_instance(self, @@ -977,7 +974,7 @@ class BlockchainDeployerInterface(BlockchainInterface): wrapped_contract = self.client.w3.eth.contract(abi=target_contract.abi, address=wrapper_contract.address, version=target_contract.version, - ContractFactoryClass=self._contract_factory) + ContractFactoryClass=self._CONTRACT_FACTORY) return wrapped_contract @validate_checksum_address @@ -994,7 +991,7 @@ class BlockchainDeployerInterface(BlockchainInterface): proxy_contract = self.client.w3.eth.contract(abi=abi, address=address, version=version, - ContractFactoryClass=self._contract_factory) + ContractFactoryClass=self._CONTRACT_FACTORY) # Read this dispatchers target address from the blockchain proxy_live_target_address = proxy_contract.functions.target().call() diff --git a/nucypher/blockchain/eth/multisig.py b/nucypher/blockchain/eth/multisig.py index b0bea0331..903a3f49d 100644 --- a/nucypher/blockchain/eth/multisig.py +++ b/nucypher/blockchain/eth/multisig.py @@ -87,7 +87,7 @@ class Proposal: contract = blockchain.client.w3.eth.contract(abi=abi, address=address, version=version, - ContractFactoryClass=blockchain._contract_factory) + ContractFactoryClass=blockchain._CONTRACT_FACTORY) contract_function, params = contract.decode_function_input(self.data) return contract_function, params else: diff --git a/nucypher/blockchain/eth/sol/__conf__.py b/nucypher/blockchain/eth/sol/__conf__.py index f3e01ee4a..652e8feca 100644 --- a/nucypher/blockchain/eth/sol/__conf__.py +++ b/nucypher/blockchain/eth/sol/__conf__.py @@ -15,4 +15,4 @@ You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ -SOLIDITY_COMPILER_VERSION = 'v0.7.3' +SOLIDITY_COMPILER_VERSION = 'v0.7.5' diff --git a/nucypher/blockchain/eth/sol/compile.py b/nucypher/blockchain/eth/sol/compile.py deleted file mode 100644 index 3c89c90dc..000000000 --- a/nucypher/blockchain/eth/sol/compile.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -This file is part of nucypher. - -nucypher is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -nucypher is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with nucypher. If not, see . -""" - - -from os.path import abspath, dirname - -import itertools -import os -import re -from typing import List, NamedTuple, Optional, Set - -from nucypher.blockchain.eth.sol import SOLIDITY_COMPILER_VERSION -from nucypher.utilities.logging import Logger - - -class SourceDirs(NamedTuple): - root_source_dir: str - other_source_dirs: Optional[Set[str]] = None - - -class SolidityCompiler: - - __default_contract_version = 'v0.0.0' - __default_contract_dir = os.path.join(dirname(abspath(__file__)), 'source') - - __compiled_contracts_dir = 'contracts' - __zeppelin_library_dir = 'zeppelin' - __aragon_library_dir = 'aragon' - - optimization_runs = 200 - - class CompilerError(Exception): - pass - - class VersionError(Exception): - pass - - @classmethod - def default_contract_dir(cls): - return cls.__default_contract_dir - - def __init__(self, - source_dirs: List[SourceDirs] = None, - ignore_solidity_check: bool = False - ) -> None: - - # Allow for optional installation - from solcx.install import get_executable - - self.log = Logger('solidity-compiler') - - version = SOLIDITY_COMPILER_VERSION if not ignore_solidity_check else None - self.__sol_binary_path = get_executable(version=version) - - if source_dirs is None or len(source_dirs) == 0: - self.source_dirs = [SourceDirs(root_source_dir=self.__default_contract_dir)] - else: - self.source_dirs = source_dirs - - def compile(self) -> dict: - interfaces = dict() - for root_source_dir, other_source_dirs in self.source_dirs: - if root_source_dir is None: - self.log.warn("One of the root directories is None") - continue - - raw_interfaces = self._compile(root_source_dir, other_source_dirs) - for name, data in raw_interfaces.items(): - # Extract contract version from docs - version_search = re.search(r""" - - \"details\": # @dev tag in contract docs - \".*? # Skip any data in the beginning of details - \| # Beginning of version definition | - (v # Capture version starting from symbol v - \d+ # At least one digit of major version - \. # Digits splitter - \d+ # At least one digit of minor version - \. # Digits splitter - \d+ # At least one digit of patch - ) # End of capturing - \| # End of version definition | - .*?\" # Skip any data in the end of details - - """, data['devdoc'], re.VERBOSE) - version = version_search.group(1) if version_search else self.__default_contract_version - try: - existence_data = interfaces[name] - except KeyError: - existence_data = dict() - interfaces.update({name: existence_data}) - if version not in existence_data: - existence_data.update({version: data}) - return interfaces - - def _compile(self, root_source_dir: str, other_source_dirs: [str]) -> dict: - """Executes the compiler with parameters specified in the json config""" - - # Allow for optional installation - from solcx import compile_files - from solcx.exceptions import SolcError - - self.log.info("Using solidity compiler binary at {}".format(self.__sol_binary_path)) - contracts_dir = os.path.join(root_source_dir, self.__compiled_contracts_dir) - self.log.info("Compiling solidity source files at {}".format(contracts_dir)) - - source_paths = set() - source_walker = os.walk(top=contracts_dir, topdown=True) - if other_source_dirs is not None: - for source_dir in other_source_dirs: - other_source_walker = os.walk(top=source_dir, topdown=True) - source_walker = itertools.chain(source_walker, other_source_walker) - - for root, dirs, files in source_walker: - for filename in files: - if filename.endswith('.sol'): - path = os.path.join(root, filename) - source_paths.add(path) - self.log.debug("Collecting solidity source {}".format(path)) - - # Compile with remappings: https://github.com/ethereum/py-solc - zeppelin_dir = os.path.join(root_source_dir, self.__zeppelin_library_dir) - aragon_dir = os.path.join(root_source_dir, self.__aragon_library_dir) - - remappings = ("contracts={}".format(contracts_dir), - "zeppelin={}".format(zeppelin_dir), - "aragon={}".format(aragon_dir), - ) - - self.log.info("Compiling with import remappings {}".format(", ".join(remappings))) - - optimization_runs = self.optimization_runs - - try: - compiled_sol = compile_files(source_files=source_paths, - solc_binary=self.__sol_binary_path, - import_remappings=remappings, - allow_paths=root_source_dir, - optimize=True, - optimize_runs=optimization_runs) - - self.log.info("Successfully compiled {} contracts with {} optimization runs".format(len(compiled_sol), - optimization_runs)) - - except FileNotFoundError: - raise RuntimeError("The solidity compiler is not at the specified path. " - "Check that the file exists and is executable.") - except PermissionError: - raise RuntimeError("The solidity compiler binary at {} is not executable. " - "Check the file's permissions.".format(self.__sol_binary_path)) - - except SolcError: - raise - - # Cleanup the compiled data keys - interfaces = {name.split(':')[-1]: compiled_sol[name] for name in compiled_sol} - return interfaces diff --git a/nucypher/blockchain/eth/sol/compile/__init__.py b/nucypher/blockchain/eth/sol/compile/__init__.py new file mode 100644 index 000000000..7d0a5d13e --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/__init__.py @@ -0,0 +1,16 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" diff --git a/nucypher/blockchain/eth/sol/compile/aggregation.py b/nucypher/blockchain/eth/sol/compile/aggregation.py new file mode 100644 index 000000000..3e7f2ddfb --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/aggregation.py @@ -0,0 +1,162 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" + + +import re +from re import Pattern +from typing import Dict + +from cytoolz.dicttoolz import merge + +from nucypher.blockchain.eth.sol.compile.constants import DEFAULT_VERSION_STRING, SOLC_LOGGER +from nucypher.blockchain.eth.sol.compile.exceptions import CompilationError, ProgrammingError +from nucypher.blockchain.eth.sol.compile.types import VersionedContractOutputs, CompiledContractOutputs + +# RE pattern for matching solidity source compile version specification in devdoc details. +DEVDOC_VERSION_PATTERN: Pattern = re.compile(r""" +\A # Anchor must be first +\| # Anchored pipe literal at beginning of version definition +( # Start Inner capture group +v # Capture version starting from symbol v +\d+ # At least one digit of major version +\. # Digits splitter +\d+ # At least one digit of minor version +\. # Digits splitter +\d+ # At least one digit of patch +) # End of capturing +\| # Anchored end of version definition | +\Z # Anchor must be the end of the match +""", re.VERBOSE) + +# simplified version of pattern to extract metadata hash from bytecode +# see https://docs.soliditylang.org/en/latest/metadata.html#encoding-of-the-metadata-hash-in-the-bytecode +METADATA_HASH_PATTERN: Pattern = re.compile(r""" +a2 +64 +69706673 # 'i' 'p' 'f' 's' +58 +22 +\w{68} # 34 bytes IPFS hash +64 +736f6c63 # 's' 'o' 'l' 'c' +43 +\w{6} # <3 byte version encoding> +0033 +""", re.VERBOSE) + + +def extract_version(compiled_contract_outputs: dict) -> str: + """ + Returns the source specified version of a compiled solidity contract. + Examines compiled contract output for devdoc details and perform a fulltext search for a source version specifier. + """ + try: + devdoc: Dict[str, str] = compiled_contract_outputs['devdoc'] + except KeyError: + # Edge Case + # --------- + # If this block is reached, the compiler did not produce results for devdoc at all. + # Ensure 'devdoc' is listed in `CONTRACT_OUTPUTS` and that solc is the latest version. + raise CompilationError(f'Solidity compiler did not output devdoc.' + f'Check the contract output compiler settings.') + else: + title = devdoc.get('title', '') + + try: + devdoc_details: str = devdoc['details'] + except KeyError: + # This is acceptable behaviour, most likely an un-versioned contract + SOLC_LOGGER.debug(f'No solidity source version specified.') + return DEFAULT_VERSION_STRING + + # RE Full Match + raw_matches = DEVDOC_VERSION_PATTERN.fullmatch(devdoc_details) + + # Positive match(es) + if raw_matches: + matches = raw_matches.groups() + if len(matches) != 1: # sanity check + # Severe Edge Case + # ----------------- + # "Impossible" situation: If this block is ever reached, + # the regular expression matching contract versions + # inside devdoc details matched multiple groups (versions). + # If you are here, and this exception is raised - do not panic! + # This most likely means there is a programming error + # in the `VERSION_PATTERN` regular expression or the surrounding logic. + raise ProgrammingError(f"Multiple version matches in {title} devdoc.") + version = matches[0] # good match + return version # OK + else: + # Negative match: Devdoc included without a version + SOLC_LOGGER.debug(f"Contract {title} not versioned.") + return DEFAULT_VERSION_STRING + + +def validate_merge(existing_version: CompiledContractOutputs, + new_version: CompiledContractOutputs, + version_specifier: str) -> None: + """Compare with incoming compiled contract data""" + new_title = new_version['devdoc'].get('title') + versioned: bool = version_specifier != DEFAULT_VERSION_STRING + if versioned and new_title: + existing_title = existing_version['devdoc'].get('title') + if existing_title == new_title: # This is the same contract + # TODO this code excludes hash of metadata, it's not perfect because format of metadata could change + # ideally use a proper CBOR parser + existing_bytecode = METADATA_HASH_PATTERN.sub('', existing_version['evm']['bytecode']['object']) + new_bytecode = METADATA_HASH_PATTERN.sub('', new_version['evm']['bytecode']['object']) + if not existing_bytecode == new_bytecode: + message = f"Two solidity sources ({new_title}, {existing_title}) specify version '{version_specifier}' " \ + "but have different compiled bytecode. Ensure that the devdoc version is " \ + "accurately updated before trying again." + raise CompilationError(message) + + +def merge_contract_sources(*compiled_sources): + return merge(*compiled_sources) # TODO: Handle file-level output aggregation + + +def merge_contract_outputs(*compiled_versions) -> VersionedContractOutputs: + versioned_outputs = dict() + + for bundle in compiled_versions: + + for contract_outputs in bundle: + version = extract_version(compiled_contract_outputs=contract_outputs) + + try: + existing_version = versioned_outputs[version] + + except KeyError: + # New Version Entry + bytecode = contract_outputs['evm']['bytecode']['object'] + for existing_version, existing_contract_outputs in versioned_outputs.items(): + existing_bytecode = existing_contract_outputs['evm']['bytecode']['object'] + if bytecode == existing_bytecode: + raise CompilationError(f"Two solidity sources compiled identical bytecode for version {version}. " + "Ensure the correct solidity paths are targeted for compilation.") + versioned_outputs[version] = contract_outputs + + else: + # Existing Version Update + validate_merge(existing_version=existing_version, + new_version=contract_outputs, + version_specifier=version) + versioned_outputs[version] = contract_outputs + + return VersionedContractOutputs(versioned_outputs) diff --git a/nucypher/blockchain/eth/sol/compile/collect.py b/nucypher/blockchain/eth/sol/compile/collect.py new file mode 100644 index 000000000..3dc918977 --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/collect.py @@ -0,0 +1,55 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" + +from nucypher.blockchain.eth.sol.compile.types import SourceBundle +from nucypher.exceptions import DevelopmentInstallationRequired +try: + import tests +except ImportError: + raise DevelopmentInstallationRequired(importable_name='tests') +from nucypher.blockchain.eth.sol.compile.constants import IGNORE_CONTRACT_PREFIXES, SOLC_LOGGER +import os +from pathlib import Path +from typing import Dict, Iterator + + +def source_filter(filename: str) -> bool: + """Helper function for filtering out contracts not intended for compilation""" + contains_ignored_prefix: bool = any(prefix in filename for prefix in IGNORE_CONTRACT_PREFIXES) + is_solidity_file: bool = filename.endswith('.sol') + return is_solidity_file and not contains_ignored_prefix + + +def collect_sources(source_bundle: SourceBundle) -> Dict[str, Path]: + """ + Combines sources bundle paths. Walks source_dir top-down to the bottom filepath of + each subdirectory recursively nd filtrates by __source_filter, setting values into `source_paths`. + """ + source_paths = dict() + combined_paths = (source_bundle.base_path, *source_bundle.other_paths) + for source_dir in combined_paths: + source_walker: Iterator = os.walk(top=source_dir, topdown=True) + for root, dirs, files in source_walker: # Collect single directory + for filename in filter(source_filter, files): # Collect files in source dir + path = Path(root) / filename + if filename in source_paths: + raise RuntimeError(f'"{filename}" source is already collected. Verify source bundle filepaths.' + f' Existing {source_paths[filename]}; Duplicate {path}.') + source_paths[filename] = path + SOLC_LOGGER.debug(f"Collecting solidity source {path}") + SOLC_LOGGER.info(f"Collected {len(source_paths)} solidity source files at {source_bundle}") + return source_paths diff --git a/nucypher/blockchain/eth/sol/compile/compile.py b/nucypher/blockchain/eth/sol/compile/compile.py new file mode 100644 index 000000000..86f3af51d --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/compile.py @@ -0,0 +1,82 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" + + +import itertools +from pathlib import Path +from typing import Tuple, List, Dict, Optional + +from cytoolz.dicttoolz import merge, merge_with + +from nucypher.blockchain.eth.sol.__conf__ import SOLIDITY_COMPILER_VERSION +from nucypher.blockchain.eth.sol.compile.aggregation import merge_contract_outputs +from nucypher.blockchain.eth.sol.compile.collect import collect_sources +from nucypher.blockchain.eth.sol.compile.config import BASE_COMPILER_CONFIGURATION, REMAPPINGS +from nucypher.blockchain.eth.sol.compile.solc import __execute +from nucypher.blockchain.eth.sol.compile.types import ( + VersionString, + VersionedContractOutputs, + CompiledContractOutputs, + SourceBundle +) + +CompilerSources = Dict[str, Dict[str, List[str]]] + + +def prepare_source_configuration(sources: Dict[str, Path]) -> CompilerSources: + input_sources = dict() + for source_name, path in sources.items(): + source_url = path.resolve(strict=True) # require source path existence + input_sources[source_name] = dict(urls=[str(source_url)]) + return input_sources + + +def prepare_remappings_configuration(base_path: Path) -> Dict: + remappings_array = list() + for i, value in enumerate(REMAPPINGS): + remappings_array.append(f"{value}={str(base_path / value)}") + remappings = dict(remappings=remappings_array) + return remappings + + +def compile_sources(source_bundle: SourceBundle, version_check: bool = True) -> Dict: + """Compiled solidity contracts for a single source directory""" + sources = collect_sources(source_bundle=source_bundle) + source_config = prepare_source_configuration(sources=sources) + solc_configuration = merge(BASE_COMPILER_CONFIGURATION, dict(sources=source_config)) # does not mutate. + + remappings_config = prepare_remappings_configuration(base_path=source_bundle.base_path) + solc_configuration['settings'].update(remappings_config) + + version: VersionString = VersionString(SOLIDITY_COMPILER_VERSION) if version_check else None + allow_paths = [source_bundle.base_path, *source_bundle.other_paths] + compiler_output = __execute(compiler_version=version, input_config=solc_configuration, allow_paths=allow_paths) + return compiler_output + + +def multiversion_compile(source_bundles: List[SourceBundle], + compiler_version_check: bool = True + ) -> VersionedContractOutputs: + """Compile contracts from `source_dirs` and aggregate the resulting source contract outputs by version""" + raw_compiler_results: List[CompiledContractOutputs] = list() + for bundle in source_bundles: + compile_result = compile_sources(source_bundle=bundle, + version_check=compiler_version_check) + raw_compiler_results.append(compile_result['contracts']) + raw_compiled_contracts = itertools.chain.from_iterable(output.values() for output in raw_compiler_results) + versioned_contract_outputs = VersionedContractOutputs(merge_with(merge_contract_outputs, *raw_compiled_contracts)) + return versioned_contract_outputs diff --git a/nucypher/blockchain/eth/sol/compile/config.py b/nucypher/blockchain/eth/sol/compile/config.py new file mode 100644 index 000000000..320fc8f4f --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/config.py @@ -0,0 +1,124 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" + + +from typing import List, Dict + +from nucypher.blockchain.eth.sol.compile.constants import ( + CONTRACTS, ZEPPELIN, ARAGON +) +from nucypher.blockchain.eth.sol.compile.types import CompilerConfiguration + +""" +Standard "JSON I/O" Compiler Config Reference: + +Input: https://solidity.readthedocs.io/en/latest/using-the-compiler.html#input-description +Output: https://solidity.readthedocs.io/en/latest/using-the-compiler.html#output-description + +WARNING: Do not change these values unless you know what you are doing. +""" + + +# Debug +# ----- +# How to treat revert (and require) reason strings. +# "default", "strip", "debug" and "verboseDebug". +# "default" does not inject compiler-generated revert strings and keeps user-supplied ones. +# "strip" removes all revert strings (if possible, i.e. if literals are used) keeping side-effects +# "debug" injects strings for compiler-generated internal reverts, implemented for ABI encoders V1 and V2 for now. +# "verboseDebug" even appends further information to user-supplied revert strings (not yet implemented) +# DEBUG = 'default' + +# Source code language. Currently supported are "Solidity" and "Yul". +LANGUAGE: str = 'Solidity' + +# Version of the EVM to compile for. Affects type checking and code generation. +EVM_VERSION: str = 'berlin' + +# File level compiler outputs (needs empty string as contract name): +FILE_OUTPUTS: List[str] = [ + 'ast' # AST of all source files + # 'legacyAST' # legacy AST of all source files +] + +# Contract level (needs the contract name or "*") +CONTRACT_OUTPUTS: List[str] = [ + + 'abi', # ABI + 'devdoc', # Developer documentation (natspec) + 'userdoc', # User documentation (natspec) + 'evm.bytecode.object', # Bytecode object + + # 'metadata', # Metadata + # 'ir', # Yul intermediate representation of the code before optimization + # 'irOptimized', # Intermediate representation after optimization + # 'storageLayout', # Slots, offsets and types of the contract's state variables. + # 'evm.assembly', # New assembly format + # 'evm.legacyAssembly', # Old-style assembly format in JSON + # 'evm.bytecode.opcodes', # Opcodes list + # 'evm.bytecode.sourceMap', # Source mapping (useful for debugging) + # 'evm.bytecode.linkReferences', # Link references (if unlinked object) + # 'evm.deployedBytecode*', # Deployed bytecode (has all the options that evm.bytecode has) + # 'evm.deployedBytecode.immutableReferences', # Map from AST ids to bytecode ranges that reference immutables + # 'evm.methodIdentifiers', # The list of function hashes + # 'evm.gasEstimates', # Function gas estimates + # 'ewasm.wast', # eWASM S-expressions format (not supported at the moment) + # 'ewasm.wasm', # eWASM binary format (not supported at the moment) +] + +# Optimizer Details - Switch optimizer components on or off in detail. +# The "enabled" switch above provides two defaults which can be tweaked here (yul, and ...). +OPTIMIZER_DETAILS = dict( + peephole=True, # The peephole optimizer is always on if no details are given (switch it off here). + jumpdestRemover=True, # The unused jumpdest remover is always on if no details are given (switch it off here). + orderLiterals=False, # Sometimes re-orders literals in commutative operations. + deduplicate=False, # Removes duplicate code blocks + cse=False, # Common subexpression elimination, Most complicated step but provides the largest gain. + constantOptimizer=False, # Optimize representation of literal numbers and strings in code. + + # The new Yul optimizer. Mostly operates on the code of ABIEncoderV2 and inline assembly. + # It is activated together with the global optimizer setting and can be deactivated here. + # Before Solidity 0.6.0 it had to be activated through this switch. Also see 'yulDetails options'. + yul=True +) + +# Optimize for how many times you intend to run the code. +# Lower values will optimize more for initial deployment cost, higher +# values will optimize more for high-frequency usage. +OPTIMIZER_RUNS = 200 + +OPTIMIZER_SETTINGS = dict( + enabled=True, + runs=OPTIMIZER_RUNS, + # details=OPTIMIZER_DETAILS # Optional - If "details" is given, "enabled" can be omitted. +) + +# Complete compiler settings +COMPILER_SETTINGS: Dict = dict( + optimizer=OPTIMIZER_SETTINGS, + evmVersion=EVM_VERSION, + outputSelection={"*": {"*": CONTRACT_OUTPUTS, "": FILE_OUTPUTS}}, # all contacts(*), all files("") +) + +REMAPPINGS: List = [CONTRACTS, ZEPPELIN, ARAGON] + +# Base configuration for programmatic usage +BASE_COMPILER_CONFIGURATION = CompilerConfiguration( + language=LANGUAGE, + settings=COMPILER_SETTINGS, + # sources and remappings added dynamically during runtime +) diff --git a/nucypher/blockchain/eth/sol/compile/constants.py b/nucypher/blockchain/eth/sol/compile/constants.py new file mode 100644 index 000000000..e6b2d69d3 --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/constants.py @@ -0,0 +1,44 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" + +from pathlib import Path +from typing import Tuple +from nucypher.config.constants import NUCYPHER_TEST_DIR + +# Logging +from nucypher.utilities.logging import Logger + +SOLC_LOGGER = Logger("solidity-compilation") + +# Vocabulary +CONTRACTS = 'contracts' + +TEST_SOLIDITY_SOURCE_ROOT: Path = Path(NUCYPHER_TEST_DIR) / CONTRACTS / CONTRACTS +TEST_MULTIVERSION_CONTRACTS: Path = Path(NUCYPHER_TEST_DIR) / 'acceptance' / 'blockchain' / 'interfaces' / 'test_contracts' / 'multiversion' + +from nucypher.blockchain.eth import sol +SOLIDITY_SOURCE_ROOT: Path = Path(sol.__file__).parent / 'source' +ZEPPELIN = 'zeppelin' +ARAGON = 'aragon' + +# Do not compile contracts containing... +IGNORE_CONTRACT_PREFIXES: Tuple[str, ...] = ( + 'Abstract', + 'Interface' +) + +DEFAULT_VERSION_STRING: str = 'v0.0.0' # for both compiler and devdoc versions (must fully match regex pattern below) diff --git a/nucypher/blockchain/eth/sol/compile/exceptions.py b/nucypher/blockchain/eth/sol/compile/exceptions.py new file mode 100644 index 000000000..78318ddbc --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/exceptions.py @@ -0,0 +1,27 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" + + +class CompilationError(RuntimeError): + """ + Raised when there is a problem compiling nucypher contracts + or with the expected compiler configuration. + """ + + +class ProgrammingError(RuntimeError): + """Caused by a human error in code""" diff --git a/nucypher/blockchain/eth/sol/compile/solc.py b/nucypher/blockchain/eth/sol/compile/solc.py new file mode 100644 index 000000000..4775243f8 --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/solc.py @@ -0,0 +1,55 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" +from pathlib import Path +from typing import Dict, Optional, List + +from nucypher.blockchain.eth.sol.compile.config import OPTIMIZER_RUNS +from nucypher.blockchain.eth.sol.compile.constants import SOLC_LOGGER +from nucypher.blockchain.eth.sol.compile.exceptions import CompilationError +from nucypher.blockchain.eth.sol.compile.types import VersionString +from nucypher.exceptions import DevelopmentInstallationRequired + + +def __execute(compiler_version: VersionString, input_config: Dict, allow_paths: Optional[List[str]]): + """Executes the solcx command and underlying solc wrapper""" + + # Lazy import to allow for optional installation of solcx + try: + from solcx.install import get_executable + from solcx.main import compile_standard + except ImportError: + raise DevelopmentInstallationRequired(importable_name='solcx') + + # Prepare Solc Command + solc_binary_path: Path = get_executable(version=compiler_version) + SOLC_LOGGER.info(f"Compiling with base path") # TODO: Add base path + + _allow_paths = ',' + ','.join(str(p) for p in allow_paths) + + # Execute Compilation + try: + compiler_output = compile_standard(input_data=input_config, + allow_paths=_allow_paths, + solc_binary=solc_binary_path) + except FileNotFoundError: + raise CompilationError("The solidity compiler is not at the specified path. " + "Check that the file exists and is executable.") + except PermissionError: + raise CompilationError(f"The solidity compiler binary at {solc_binary_path} is not executable. " + "Check the file's permissions.") + SOLC_LOGGER.info(f"Successfully compiled {len(compiler_output)} sources with {OPTIMIZER_RUNS} optimization runs") + return compiler_output diff --git a/nucypher/blockchain/eth/sol/compile/types.py b/nucypher/blockchain/eth/sol/compile/types.py new file mode 100644 index 000000000..07c20e5a5 --- /dev/null +++ b/nucypher/blockchain/eth/sol/compile/types.py @@ -0,0 +1,48 @@ +""" +This file is part of nucypher. + +nucypher is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +nucypher is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with nucypher. If not, see . +""" +from pathlib import Path +from typing import Dict, List, Union, NewType, NamedTuple, Tuple, Optional + + +class ABI(Dict): + inputs: List + name: str + outputs: List[Dict[str, str]] + stateMutability: str + type: str + + +class CompiledContractOutputs(Dict): + abi: ABI + devdoc: Dict[str, Union[str, Dict[str, str]]] + evm: Dict[str, Dict] + userdoc: Dict + + +VersionString = NewType('VersionString', str) +VersionedContractOutputs = NewType('VersionedContractOutputs', Dict[VersionString, CompiledContractOutputs]) + + +class CompilerConfiguration(Dict): + language: str + sources: Dict[str, Dict[str, str]] + settings: Dict + + +class SourceBundle(NamedTuple): + base_path: Path + other_paths: Tuple[Path, ...] = tuple() diff --git a/nucypher/characters/lawful.py b/nucypher/characters/lawful.py index 475189e46..a9b2dd6d1 100644 --- a/nucypher/characters/lawful.py +++ b/nucypher/characters/lawful.py @@ -229,7 +229,7 @@ class Alice(Character, BlockchainPolicyAuthor): policy = FederatedPolicy(alice=self, **payload) else: - # Sample from blockchain via PolicyManager + # Sample from blockchain PolicyManager from nucypher.policy.policies import BlockchainPolicy payload.update(**policy_params) policy = BlockchainPolicy(alice=self, **payload) diff --git a/nucypher/cli/commands/multisig.py b/nucypher/cli/commands/multisig.py index 56c81dc40..15a2ba875 100644 --- a/nucypher/cli/commands/multisig.py +++ b/nucypher/cli/commands/multisig.py @@ -248,7 +248,7 @@ def sign(general_config, blockchain_options, multisig_options, proposal): proxy_contract = blockchain.client.w3.eth.contract(abi=abi, address=address, version=version, - ContractFactoryClass=blockchain._contract_factory) + ContractFactoryClass=blockchain._CONTRACT_FACTORY) paint_multisig_proposed_transaction(emitter, proposal, proxy_contract) click.confirm(PROMPT_CONFIRM_MULTISIG_SIGNATURE, abort=True) @@ -290,7 +290,7 @@ def execute(general_config, blockchain_options, multisig_options, proposal): proxy_contract = blockchain.client.w3.eth.contract(abi=abi, address=address, version=version, - ContractFactoryClass=blockchain._contract_factory) + ContractFactoryClass=blockchain._CONTRACT_FACTORY) paint_multisig_proposed_transaction(emitter, proposal, proxy_contract) trustee = multisig_options.create_trustee(registry) diff --git a/nucypher/config/constants.py b/nucypher/config/constants.py index 7423a7a9b..93a9fc010 100644 --- a/nucypher/config/constants.py +++ b/nucypher/config/constants.py @@ -16,13 +16,15 @@ along with nucypher. If not, see . """ -from collections import namedtuple - import os -from appdirs import AppDirs +from collections import namedtuple +from os.path import dirname from pathlib import Path +from appdirs import AppDirs + import nucypher +from nucypher.exceptions import DevelopmentInstallationRequired # Environment variables NUCYPHER_ENVVAR_KEYRING_PASSWORD = "NUCYPHER_KEYRING_PASSWORD" @@ -38,6 +40,17 @@ NUCYPHER_PACKAGE = Path(nucypher.__file__).parent.resolve() BASE_DIR = NUCYPHER_PACKAGE.parent.resolve() DEPLOY_DIR = BASE_DIR / 'deploy' +# Test Filepaths +try: + import tests +except ImportError: + raise DevelopmentInstallationRequired(importable_name='tests') +else: + # TODO: Another way to handle this situation? + # __file__ can be None, especially with namespace packages on + # Python 3.7 or when using apidoc and sphinx-build. + file_path = tests.__file__ + NUCYPHER_TEST_DIR = dirname(file_path) if file_path is not None else str() # User Application Filepaths APP_DIR = AppDirs(nucypher.__title__, nucypher.__author__) diff --git a/nucypher/types.py b/nucypher/types.py index 3e6cc2b6a..540a1036d 100644 --- a/nucypher/types.py +++ b/nucypher/types.py @@ -16,8 +16,9 @@ """ -from eth_typing.evm import ChecksumAddress from typing import TypeVar, NewType, Tuple, NamedTuple, Union + +from eth_typing.evm import ChecksumAddress from web3.types import Wei, Timestamp, TxReceipt NuNits = NewType("NuNits", int) @@ -29,11 +30,7 @@ Evidence = TypeVar('Evidence', bound='IndisputableEvidence') ContractReturnValue = TypeVar('ContractReturnValue', bound=Union[TxReceipt, Wei, int, str, bool]) -class ContractParams(Tuple): - pass - - -class WorklockParameters(ContractParams): +class WorklockParameters(Tuple): token_supply: NuNits start_bid_date: Timestamp end_bid_date: Timestamp @@ -43,7 +40,7 @@ class WorklockParameters(ContractParams): min_allowed_bid: Wei -class StakingEscrowParameters(ContractParams): +class StakingEscrowParameters(Tuple): seconds_per_period: int minting_coefficient: int lock_duration_coefficient_1: int diff --git a/tests/acceptance/blockchain/actors/test_deployer.py b/tests/acceptance/blockchain/actors/test_deployer.py index 535c8be86..1ddecaf47 100644 --- a/tests/acceptance/blockchain/actors/test_deployer.py +++ b/tests/acceptance/blockchain/actors/test_deployer.py @@ -22,21 +22,18 @@ import pytest from nucypher.blockchain.eth.actors import ContractAdministrator from nucypher.blockchain.eth.agents import StakingEscrowAgent, ContractAgency -from nucypher.blockchain.eth.sol.compile import SolidityCompiler from nucypher.characters.control.emitters import StdoutEmitter from nucypher.crypto.powers import TransactingPower +from tests.constants import INSECURE_DEVELOPMENT_PASSWORD, NUMBER_OF_ALLOCATIONS_IN_TESTS # Prevents TesterBlockchain to be picked up by py.test as a test class from tests.utils.blockchain import TesterBlockchain as _TesterBlockchain -from tests.constants import INSECURE_DEVELOPMENT_PASSWORD, NUMBER_OF_ALLOCATIONS_IN_TESTS @pytest.mark.usefixtures('testerchain') def test_rapid_deployment(token_economics, test_registry, tmpdir, get_random_checksum_address): - compiler = SolidityCompiler() blockchain = _TesterBlockchain(eth_airdrop=False, - test_accounts=4, - compiler=compiler) + test_accounts=4) # TODO: #1092 - TransactingPower blockchain.transacting_power = TransactingPower(password=INSECURE_DEVELOPMENT_PASSWORD, diff --git a/tests/acceptance/blockchain/deployers/test_token_deployer.py b/tests/acceptance/blockchain/deployers/test_token_deployer.py index 04b3314e0..fbde60dc4 100644 --- a/tests/acceptance/blockchain/deployers/test_token_deployer.py +++ b/tests/acceptance/blockchain/deployers/test_token_deployer.py @@ -25,7 +25,7 @@ def test_token_deployer_and_agent(testerchain, deployment_progress, test_registr origin = testerchain.etherbase_account - # Trying to get token from blockchain before it's been published fails + # Trying to get token from blockchain before it's been published should fail with pytest.raises(BaseContractRegistry.UnknownContract): NucypherTokenAgent(registry=test_registry) diff --git a/tests/acceptance/blockchain/interfaces/contracts/multiversion/v1/VersionTest.sol b/tests/acceptance/blockchain/interfaces/test_contracts/multiversion/v1/VersionTest.sol similarity index 100% rename from tests/acceptance/blockchain/interfaces/contracts/multiversion/v1/VersionTest.sol rename to tests/acceptance/blockchain/interfaces/test_contracts/multiversion/v1/VersionTest.sol diff --git a/tests/acceptance/blockchain/interfaces/contracts/multiversion/v2/VersionTest.sol b/tests/acceptance/blockchain/interfaces/test_contracts/multiversion/v2/VersionTest.sol similarity index 100% rename from tests/acceptance/blockchain/interfaces/contracts/multiversion/v2/VersionTest.sol rename to tests/acceptance/blockchain/interfaces/test_contracts/multiversion/v2/VersionTest.sol diff --git a/tests/acceptance/blockchain/interfaces/test_handle_multiversion_contracts.py b/tests/acceptance/blockchain/interfaces/test_handle_multiversion_contracts.py new file mode 100644 index 000000000..4624c3499 --- /dev/null +++ b/tests/acceptance/blockchain/interfaces/test_handle_multiversion_contracts.py @@ -0,0 +1,102 @@ +""" + This file is part of nucypher. + + nucypher is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + nucypher is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with nucypher. If not, see . +""" + + +from nucypher.blockchain.eth.interfaces import BlockchainDeployerInterface, BlockchainInterfaceFactory +from nucypher.blockchain.eth.registry import InMemoryContractRegistry +from nucypher.blockchain.eth.sol.compile.constants import TEST_MULTIVERSION_CONTRACTS +from nucypher.blockchain.eth.sol.compile.types import SourceBundle +from nucypher.crypto.powers import TransactingPower +from tests.constants import INSECURE_DEVELOPMENT_PASSWORD +from tests.utils.blockchain import free_gas_price_strategy + + +def test_deployer_interface_multiversion_contract(): + + # Prepare compiler + base_dir = TEST_MULTIVERSION_CONTRACTS + v1_dir, v2_dir = base_dir / 'v1', base_dir / 'v2' + + # TODO: Check type of sources + # I am a contract administrator and I an compiling a new updated version of an existing contract... + # Represents "Manually hardcoding" a new source directory on BlockchainDeployerInterface.SOURCES. + BlockchainDeployerInterface.SOURCES = ( + SourceBundle(base_path=v1_dir), + SourceBundle(base_path=v2_dir) + ) + + # Prepare chain + BlockchainInterfaceFactory._interfaces.clear() + blockchain_interface = BlockchainDeployerInterface(provider_uri='tester://pyevm', + gas_strategy=free_gas_price_strategy) + blockchain_interface.connect() + BlockchainInterfaceFactory.register_interface(interface=blockchain_interface) # Lets this test run in isolation + + origin = blockchain_interface.client.accounts[0] + blockchain_interface.transacting_power = TransactingPower(password=INSECURE_DEVELOPMENT_PASSWORD, account=origin) + blockchain_interface.transacting_power.activate() + + # Searching both contract through raw data + contract_name = "VersionTest" + requested_version = "v1.2.3" + version, _data = blockchain_interface.find_raw_contract_data(contract_name=contract_name, + requested_version=requested_version) + assert version == requested_version + version, _data = blockchain_interface.find_raw_contract_data(contract_name=contract_name, + requested_version="latest") + assert version == requested_version + + requested_version = "v1.1.4" + version, _data = blockchain_interface.find_raw_contract_data(contract_name=contract_name, + requested_version=requested_version) + assert version == requested_version + version, _data = blockchain_interface.find_raw_contract_data(contract_name=contract_name, + requested_version="earliest") + assert version == requested_version + + # Deploy different contracts and check their versions + registry = InMemoryContractRegistry() + contract, receipt = blockchain_interface.deploy_contract(deployer_address=origin, + registry=registry, + contract_name=contract_name, + contract_version="v1.1.4") + assert contract.version == "v1.1.4" + assert contract.functions.VERSION().call() == 1 + contract, receipt = blockchain_interface.deploy_contract(deployer_address=origin, + registry=registry, + contract_name=contract_name, + contract_version="earliest") + assert contract.version == "v1.1.4" + assert contract.functions.VERSION().call() == 1 + + contract, receipt = blockchain_interface.deploy_contract(deployer_address=origin, + registry=registry, + contract_name=contract_name, + contract_version="v1.2.3") + assert contract.version == "v1.2.3" + assert contract.functions.VERSION().call() == 2 + contract, receipt = blockchain_interface.deploy_contract(deployer_address=origin, + registry=registry, + contract_name=contract_name, + contract_version="latest") + assert contract.version == "v1.2.3" + assert contract.functions.VERSION().call() == 2 + contract, receipt = blockchain_interface.deploy_contract(deployer_address=origin, + registry=registry, + contract_name=contract_name) + assert contract.version == "v1.2.3" + assert contract.functions.VERSION().call() == 2 diff --git a/tests/acceptance/blockchain/interfaces/test_solidity_compiler.py b/tests/acceptance/blockchain/interfaces/test_solidity_compiler.py index f2cdabbc3..35d36aacd 100644 --- a/tests/acceptance/blockchain/interfaces/test_solidity_compiler.py +++ b/tests/acceptance/blockchain/interfaces/test_solidity_compiler.py @@ -14,54 +14,46 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ -import os -from os.path import abspath, dirname from nucypher.blockchain.eth.deployers import NucypherTokenDeployer -from nucypher.blockchain.eth.sol.compile import SolidityCompiler, SourceDirs -from tests.constants import TEST_CONTRACTS_DIR +from nucypher.blockchain.eth.sol.compile.compile import multiversion_compile +from nucypher.blockchain.eth.sol.compile.constants import DEFAULT_VERSION_STRING, TEST_MULTIVERSION_CONTRACTS, \ + TEST_SOLIDITY_SOURCE_ROOT, SOLIDITY_SOURCE_ROOT +from nucypher.blockchain.eth.sol.compile.types import SourceBundle def test_nucypher_contract_compiled(testerchain, test_registry): - # Ensure that solidity smart contacts are available, post-compile. + """Ensure that solidity smart contacts are available, post-compile.""" origin, *everybody_else = testerchain.client.accounts token_contract_identifier = NucypherTokenDeployer(registry=test_registry, deployer_address=origin).contract_name assert token_contract_identifier in testerchain._raw_contract_cache token_data = testerchain._raw_contract_cache[token_contract_identifier] assert len(token_data) == 1 - assert "v0.0.0" in token_data + assert DEFAULT_VERSION_STRING in token_data def test_multi_source_compilation(testerchain): - solidity_compiler = SolidityCompiler(source_dirs=[ - (SolidityCompiler.default_contract_dir(), None), - (SolidityCompiler.default_contract_dir(), {TEST_CONTRACTS_DIR}) - ]) - interfaces = solidity_compiler.compile() - - # Remove AST because id in tree node depends on compilation scope - for contract_name, contract_data in interfaces.items(): - for version, data in contract_data.items(): - data.pop("ast") + bundles = [ + SourceBundle(base_path=SOLIDITY_SOURCE_ROOT), + SourceBundle(base_path=SOLIDITY_SOURCE_ROOT, other_paths=(TEST_SOLIDITY_SOURCE_ROOT,)) + ] + interfaces = multiversion_compile(source_bundles=bundles) raw_cache = testerchain._raw_contract_cache.copy() - for contract_name, contract_data in raw_cache.items(): - for version, data in contract_data.items(): - data.pop("ast") assert interfaces == raw_cache def test_multi_versions(): - base_dir = os.path.join(dirname(abspath(__file__)), "contracts", "multiversion") - v1_dir = os.path.join(base_dir, "v1") - v2_dir = os.path.join(base_dir, "v2") - root_dir = SolidityCompiler.default_contract_dir() - solidity_compiler = SolidityCompiler(source_dirs=[SourceDirs(root_dir, {v1_dir}), - SourceDirs(root_dir, {v2_dir})]) - interfaces = solidity_compiler.compile() + base_dir = TEST_MULTIVERSION_CONTRACTS + v1_dir, v2_dir = base_dir / "v1", base_dir / "v2" + bundles = [ + SourceBundle(base_path=v1_dir), + SourceBundle(base_path=v2_dir) + ] + interfaces = multiversion_compile(source_bundles=bundles) assert "VersionTest" in interfaces contract_data = interfaces["VersionTest"] assert len(contract_data) == 2 assert "v1.2.3" in contract_data assert "v1.1.4" in contract_data - assert contract_data["v1.2.3"]["devdoc"] != contract_data["v1.1.4"]["devdoc"] + assert contract_data["v1.2.3"]["devdoc"]['details'] != contract_data["v1.1.4"]["devdoc"]['details'] diff --git a/tests/acceptance/blockchain/interfaces/test_chains.py b/tests/acceptance/blockchain/interfaces/test_testerchain.py similarity index 87% rename from tests/acceptance/blockchain/interfaces/test_chains.py rename to tests/acceptance/blockchain/interfaces/test_testerchain.py index e3370755e..27f127eaa 100644 --- a/tests/acceptance/blockchain/interfaces/test_chains.py +++ b/tests/acceptance/blockchain/interfaces/test_testerchain.py @@ -14,33 +14,30 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ -import types -from os.path import abspath, dirname -import os -from unittest.mock import PropertyMock - -import maya import pytest -from hexbytes import HexBytes from nucypher.blockchain.eth.clients import EthereumClient -from nucypher.blockchain.eth.interfaces import BlockchainDeployerInterface, BlockchainInterfaceFactory +from nucypher.blockchain.eth.interfaces import BlockchainDeployerInterface from nucypher.blockchain.eth.registry import InMemoryContractRegistry -from nucypher.blockchain.eth.sol.compile import SolidityCompiler, SourceDirs +from nucypher.blockchain.eth.sol.compile.compile import multiversion_compile +from nucypher.blockchain.eth.sol.compile.constants import TEST_MULTIVERSION_CONTRACTS, SOLIDITY_SOURCE_ROOT +from nucypher.blockchain.eth.sol.compile.types import SourceBundle +from nucypher.config.constants import NUCYPHER_TEST_DIR from nucypher.crypto.powers import TransactingPower +from tests.constants import ( + DEVELOPMENT_ETH_AIRDROP_AMOUNT, + NUMBER_OF_ETH_TEST_ACCOUNTS, + NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS, + NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS, INSECURE_DEVELOPMENT_PASSWORD +) # Prevents TesterBlockchain to be picked up by py.test as a test class -from tests.fixtures import _make_testerchain -from tests.mock.interfaces import MockBlockchain from tests.utils.blockchain import TesterBlockchain as _TesterBlockchain, free_gas_price_strategy -from tests.constants import (DEVELOPMENT_ETH_AIRDROP_AMOUNT, INSECURE_DEVELOPMENT_PASSWORD, - NUMBER_OF_ETH_TEST_ACCOUNTS, NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS, - NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS) @pytest.fixture() -def another_testerchain(solidity_compiler): - testerchain = _TesterBlockchain(eth_airdrop=True, free_transactions=True, light=True, compiler=solidity_compiler) +def another_testerchain(): + testerchain = _TesterBlockchain(eth_airdrop=True, free_transactions=True, light=True) testerchain.deployer_address = testerchain.etherbase_account assert testerchain.is_light yield testerchain @@ -96,19 +93,22 @@ def test_testerchain_creation(testerchain, another_testerchain): def test_multiversion_contract(): + # Prepare compiler - base_dir = os.path.join(dirname(abspath(__file__)), "contracts", "multiversion") - v1_dir = os.path.join(base_dir, "v1") - v2_dir = os.path.join(base_dir, "v2") - root_dir = SolidityCompiler.default_contract_dir() - solidity_compiler = SolidityCompiler(source_dirs=[SourceDirs(root_dir, {v2_dir}), - SourceDirs(root_dir, {v1_dir})]) + base_dir = TEST_MULTIVERSION_CONTRACTS + v1_dir, v2_dir = base_dir / 'v1', base_dir / 'v2' + bundles = [ + SourceBundle(base_path=SOLIDITY_SOURCE_ROOT, other_paths=(v1_dir,)), + SourceBundle(base_path=SOLIDITY_SOURCE_ROOT, other_paths=(v2_dir,)) + ] + compiled_contracts = multiversion_compile(source_bundles=bundles) # Prepare chain blockchain_interface = BlockchainDeployerInterface(provider_uri='tester://pyevm/2', - compiler=solidity_compiler, gas_strategy=free_gas_price_strategy) - blockchain_interface.connect() + blockchain_interface.connect(compile_now=False) + blockchain_interface._raw_contract_cache = compiled_contracts + origin = blockchain_interface.client.accounts[0] blockchain_interface.transacting_power = TransactingPower(password=INSECURE_DEVELOPMENT_PASSWORD, account=origin) blockchain_interface.transacting_power.activate() diff --git a/tests/acceptance/cli/deploy/test_deploy_cli.py b/tests/acceptance/cli/deploy/test_deploy_cli.py index 65850bd11..e9dcaf6ed 100644 --- a/tests/acceptance/cli/deploy/test_deploy_cli.py +++ b/tests/acceptance/cli/deploy/test_deploy_cli.py @@ -27,7 +27,7 @@ from nucypher.blockchain.eth.interfaces import BlockchainInterface from nucypher.blockchain.eth.networks import NetworksInventory from nucypher.blockchain.eth.registry import LocalContractRegistry from nucypher.blockchain.eth.signers import Signer -from nucypher.blockchain.eth.sol.compile import SOLIDITY_COMPILER_VERSION +from nucypher.blockchain.eth.sol import SOLIDITY_COMPILER_VERSION from nucypher.cli.commands.deploy import deploy from nucypher.config.constants import TEMPORARY_DOMAIN from tests.constants import TEST_PROVIDER_URI, YES_ENTER diff --git a/tests/constants.py b/tests/constants.py index 2a8a554db..644d596d6 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -16,16 +16,16 @@ along with nucypher. If not, see . """ import string -from random import SystemRandom - import tempfile import time from datetime import datetime from pathlib import Path +from random import SystemRandom + from web3 import Web3 from nucypher.blockchain.eth.token import NU -from nucypher.config.constants import BASE_DIR, NUCYPHER_ENVVAR_KEYRING_PASSWORD, NUCYPHER_ENVVAR_WORKER_ETH_PASSWORD +from nucypher.config.constants import NUCYPHER_ENVVAR_KEYRING_PASSWORD, NUCYPHER_ENVVAR_WORKER_ETH_PASSWORD # # Ursula @@ -57,7 +57,6 @@ NUMBER_OF_MOCK_KEYSTORE_ACCOUNTS = NUMBER_OF_ETH_TEST_ACCOUNTS # Testerchain # -TEST_CONTRACTS_DIR = Path(BASE_DIR) / 'tests' / 'contracts' / 'contracts' ONE_YEAR_IN_SECONDS = ((60 * 60) * 24) * 365 diff --git a/tests/contracts/test_contracts_upgradeability.py b/tests/contracts/test_contracts_upgradeability.py index 386f54cfe..9c98b744f 100644 --- a/tests/contracts/test_contracts_upgradeability.py +++ b/tests/contracts/test_contracts_upgradeability.py @@ -17,17 +17,20 @@ along with nucypher. If not, see . import contextlib import os -import pytest +from pathlib import Path + import requests from web3.exceptions import ValidationError from nucypher.blockchain.eth.deployers import AdjudicatorDeployer, BaseContractDeployer, NucypherTokenDeployer, \ - PolicyManagerDeployer, StakingEscrowDeployer + PolicyManagerDeployer, StakingEscrowDeployer, WorklockDeployer from nucypher.blockchain.eth.interfaces import BlockchainDeployerInterface, BlockchainInterfaceFactory from nucypher.blockchain.eth.registry import InMemoryContractRegistry -from nucypher.blockchain.eth.sol.compile import SolidityCompiler, SourceDirs +from nucypher.blockchain.eth.sol.compile.constants import SOLIDITY_SOURCE_ROOT +from nucypher.blockchain.eth.sol.compile.types import SourceBundle from nucypher.crypto.powers import TransactingPower from tests.constants import INSECURE_DEVELOPMENT_PASSWORD +from tests.fixtures import make_token_economics from tests.utils.blockchain import free_gas_price_strategy USER = "nucypher" @@ -97,14 +100,16 @@ def deploy_earliest_contract(blockchain_interface: BlockchainDeployerInterface, def test_upgradeability(temp_dir_path): # Prepare remote source for compilation download_github_dir(GITHUB_SOURCE_LINK, temp_dir_path) - solidity_compiler = SolidityCompiler(source_dirs=[SourceDirs(SolidityCompiler.default_contract_dir()), - SourceDirs(temp_dir_path)]) # Prepare the blockchain - provider_uri = 'tester://pyevm/2' + BlockchainDeployerInterface.SOURCES = [ + SourceBundle(base_path=SOLIDITY_SOURCE_ROOT), + SourceBundle(base_path=Path(temp_dir_path)) + ] + + provider_uri = 'tester://pyevm/2' # TODO: Testerchain caching Issues try: blockchain_interface = BlockchainDeployerInterface(provider_uri=provider_uri, - compiler=solidity_compiler, gas_strategy=free_gas_price_strategy) blockchain_interface.connect() origin = blockchain_interface.client.accounts[0] @@ -112,6 +117,8 @@ def test_upgradeability(temp_dir_path): blockchain_interface.transacting_power = TransactingPower(password=INSECURE_DEVELOPMENT_PASSWORD, account=origin) blockchain_interface.transacting_power.activate() + economics = make_token_economics(blockchain_interface) + # Check contracts with multiple versions raw_contracts = blockchain_interface._raw_contract_cache contract_name = AdjudicatorDeployer.contract_name @@ -127,25 +134,45 @@ def test_upgradeability(temp_dir_path): # Prepare master version of contracts and upgrade to the latest registry = InMemoryContractRegistry() - token_deployer = NucypherTokenDeployer(registry=registry, deployer_address=origin) + token_deployer = NucypherTokenDeployer(registry=registry, + deployer_address=origin, + economics=economics) token_deployer.deploy() - staking_escrow_deployer = StakingEscrowDeployer(registry=registry, deployer_address=origin) + staking_escrow_deployer = StakingEscrowDeployer(registry=registry, + deployer_address=origin, + economics=economics) deploy_earliest_contract(blockchain_interface, staking_escrow_deployer) + + policy_manager_deployer = None + if test_staking_escrow or test_policy_manager: + policy_manager_deployer = PolicyManagerDeployer(registry=registry, + deployer_address=origin, + economics=economics) + deploy_earliest_contract(blockchain_interface, policy_manager_deployer) + + adjudicator_deployer = None + if test_staking_escrow or test_adjudicator: + adjudicator_deployer = AdjudicatorDeployer(registry=registry, + deployer_address=origin, + economics=economics) + deploy_earliest_contract(blockchain_interface, adjudicator_deployer) + if test_staking_escrow: + worklock_deployer = WorklockDeployer(registry=registry, + deployer_address=origin, + economics=economics) + worklock_deployer.deploy() + # TODO prepare at least one staker before calling upgrade staking_escrow_deployer.upgrade(contract_version="latest", confirmations=0) if test_policy_manager: - policy_manager_deployer = PolicyManagerDeployer(registry=registry, deployer_address=origin) - deploy_earliest_contract(blockchain_interface, policy_manager_deployer) policy_manager_deployer.upgrade(contract_version="latest", confirmations=0) if test_adjudicator: - adjudicator_deployer = AdjudicatorDeployer(registry=registry, deployer_address=origin) - deploy_earliest_contract(blockchain_interface, adjudicator_deployer) adjudicator_deployer.upgrade(contract_version="latest", confirmations=0) finally: - # Unregister interface + # Unregister interface # TODO: Move to method? with contextlib.suppress(KeyError): del BlockchainInterfaceFactory._interfaces[provider_uri] diff --git a/tests/fixtures.py b/tests/fixtures.py index 1eb7956d2..fa2e2011d 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -23,6 +23,7 @@ import shutil import tempfile from datetime import datetime, timedelta from functools import partial +from typing import Tuple import maya import pytest @@ -52,10 +53,9 @@ from nucypher.blockchain.eth.deployers import ( from nucypher.blockchain.eth.interfaces import BlockchainInterfaceFactory from nucypher.blockchain.eth.registry import InMemoryContractRegistry, LocalContractRegistry from nucypher.blockchain.eth.signers.software import Web3Signer -from nucypher.blockchain.eth.sol.compile import SolidityCompiler from nucypher.blockchain.eth.token import NU from nucypher.characters.control.emitters import StdoutEmitter -from nucypher.characters.lawful import Bob, Enrico +from nucypher.characters.lawful import Enrico from nucypher.config.characters import ( AliceConfiguration, BobConfiguration, @@ -64,9 +64,7 @@ from nucypher.config.characters import ( ) from nucypher.config.constants import TEMPORARY_DOMAIN from nucypher.crypto.powers import TransactingPower -from nucypher.crypto.utils import canonical_address_from_umbral_key from nucypher.datastore import datastore -from nucypher.policy.collections import IndisputableEvidence, WorkOrder from nucypher.utilities.logging import GlobalLoggerSettings, Logger from tests.constants import ( @@ -110,8 +108,7 @@ from tests.utils.config import ( from tests.utils.middleware import MockRestMiddleware, MockRestMiddlewareForLargeFleetTests from tests.utils.policy import generate_random_label from tests.utils.ursula import MOCK_URSULA_STARTING_PORT, make_decentralized_ursulas, make_federated_ursulas, \ - MOCK_KNOWN_URSULAS_CACHE - + MOCK_KNOWN_URSULAS_CACHE, _mock_ursula_reencrypts test_logger = Logger("test-logger") @@ -458,13 +455,6 @@ def token_economics(testerchain): return make_token_economics(blockchain=testerchain) -@pytest.fixture(scope='session') -def solidity_compiler(): - """Doing this more than once per session will result in slower test run times.""" - compiler = SolidityCompiler() - yield compiler - - @pytest.fixture(scope='module') def test_registry(): registry = InMemoryContractRegistry() @@ -783,53 +773,6 @@ def funded_blockchain(testerchain, agency, token_economics, test_registry): # Re-Encryption # -def _mock_ursula_reencrypts(ursula, corrupt_cfrag: bool = False): - delegating_privkey = UmbralPrivateKey.gen_key() - _symmetric_key, capsule = pre._encapsulate(delegating_privkey.get_pubkey()) - signing_privkey = UmbralPrivateKey.gen_key() - signing_pubkey = signing_privkey.get_pubkey() - signer = Signer(signing_privkey) - priv_key_bob = UmbralPrivateKey.gen_key() - pub_key_bob = priv_key_bob.get_pubkey() - kfrags = pre.generate_kfrags(delegating_privkey=delegating_privkey, - signer=signer, - receiving_pubkey=pub_key_bob, - threshold=2, - N=4, - sign_delegating_key=False, - sign_receiving_key=False) - capsule.set_correctness_keys(delegating_privkey.get_pubkey(), pub_key_bob, signing_pubkey) - - ursula_pubkey = ursula.stamp.as_umbral_pubkey() - - alice_address = canonical_address_from_umbral_key(signing_pubkey) - blockhash = bytes(32) - - specification = b''.join((bytes(capsule), - bytes(ursula_pubkey), - bytes(ursula.decentralized_identity_evidence), - alice_address, - blockhash)) - - bobs_signer = Signer(priv_key_bob) - task_signature = bytes(bobs_signer(specification)) - - metadata = bytes(ursula.stamp(task_signature)) - - cfrag = pre.reencrypt(kfrags[0], capsule, metadata=metadata) - - if corrupt_cfrag: - cfrag.proof.bn_sig = CurveBN.gen_rand(capsule.params.curve) - - cfrag_signature = ursula.stamp(bytes(cfrag)) - - bob = Bob.from_public_keys(verifying_key=pub_key_bob) - task = WorkOrder.PRETask(capsule, task_signature, cfrag, cfrag_signature) - work_order = WorkOrder(bob, None, alice_address, {capsule: task}, None, ursula, blockhash) - - evidence = IndisputableEvidence(task, work_order) - return evidence - @pytest.fixture(scope='session') def mock_ursula_reencrypts(): @@ -951,10 +894,15 @@ def manual_worker(testerchain): # # TODO : Use a pytest Flag to enable/disable this functionality +test_logger = Logger("test-logger") + + @pytest.fixture(autouse=True, scope='function') def log_in_and_out_of_test(request): test_name = request.node.name module_name = request.module.__name__ + + test_logger.info(f"Starting {module_name}.py::{test_name}") yield test_logger.info(f"Finalized {module_name}.py::{test_name}") diff --git a/tests/metrics/estimate_gas.py b/tests/metrics/estimate_gas.py index 2ac39cfa0..279396599 100755 --- a/tests/metrics/estimate_gas.py +++ b/tests/metrics/estimate_gas.py @@ -18,32 +18,38 @@ You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ -import json -from os.path import abspath, dirname - import io +import json import os import re -import sys -import tabulate import time +from os.path import abspath, dirname +from unittest.mock import Mock + +import tabulate from twisted.logger import ILogObserver, globalLogPublisher, jsonFileLogObserver from umbral.keys import UmbralPrivateKey from umbral.signing import Signer -from unittest.mock import Mock from zope.interface import provider +from nucypher.exceptions import DevelopmentInstallationRequired from nucypher.blockchain.economics import StandardTokenEconomics -from nucypher.blockchain.eth.agents import AdjudicatorAgent, NucypherTokenAgent, PolicyManagerAgent, StakingEscrowAgent +from nucypher.blockchain.eth.agents import ( + AdjudicatorAgent, + NucypherTokenAgent, + PolicyManagerAgent, + StakingEscrowAgent +) from nucypher.blockchain.eth.constants import NUCYPHER_CONTRACT_NAMES from nucypher.crypto.signing import SignatureStamp from nucypher.policy.policies import Policy from nucypher.utilities.logging import Logger from tests.utils.blockchain import TesterBlockchain -# FIXME: Needed to use a fixture here, but now estimate_gas.py only runs if executed from main directory -sys.path.insert(0, abspath('tests')) -from fixtures import _mock_ursula_reencrypts as mock_ursula_reencrypts +try: + from tests.utils.ursula import _mock_ursula_reencrypts as mock_ursula_reencrypts +except ImportError: + raise DevelopmentInstallationRequired(importable_name='tests.utils.ursula') ALGORITHM_SHA256 = 1 @@ -176,7 +182,7 @@ def estimate_gas(analyzer: AnalyzeGas = None) -> None: compiled_contract = testerchain._raw_contract_cache[contract_name] version = list(compiled_contract).pop() - bin_runtime = compiled_contract[version]['bin-runtime'] + bin_runtime = compiled_contract[version]['evm']['bytecode']['object'] bin_length_in_bytes = len(bin_runtime) // 2 percentage = int(100 * bin_length_in_bytes / MAX_SIZE) bar = ('*'*(percentage//2)).ljust(50) diff --git a/tests/mock/interfaces.py b/tests/mock/interfaces.py index 2e1398f9e..1c7dbf5e6 100644 --- a/tests/mock/interfaces.py +++ b/tests/mock/interfaces.py @@ -73,14 +73,18 @@ def mock_registry_source_manager(blockchain, test_registry, mock_backend: bool = class MockBlockchain(TesterBlockchain): - _PROVIDER_URI = MOCK_PROVIDER_URI - _compiler = None + PROVIDER_URI = MOCK_PROVIDER_URI def __init__(self): - super().__init__(mock_backend=True) + super().__init__() class MockEthereumClient(EthereumClient): def __init__(self, w3): - super().__init__(w3, None, None, None, None) + super().__init__(w3=w3, node_technology=None, version=None, platform=None, backend=None) + + def connect(self, *args, **kwargs) -> bool: + if 'compile_now' in kwargs: + raise ValueError("Mock testerchain cannot handle solidity source compilation.") + return super().connect(compile_now=False, *args, **kwargs) diff --git a/tests/unit/test_contract_versioning.py b/tests/unit/test_contract_versioning.py new file mode 100644 index 000000000..5d1917409 --- /dev/null +++ b/tests/unit/test_contract_versioning.py @@ -0,0 +1,57 @@ +""" + This file is part of nucypher. + + nucypher is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + nucypher is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with nucypher. If not, see . +""" + +import re +from hypothesis import given, example, settings +from hypothesis import strategies + +from nucypher.blockchain.eth.sol.compile.aggregation import DEVDOC_VERSION_PATTERN +from nucypher.blockchain.eth.sol.compile.constants import DEFAULT_VERSION_STRING + + +@example('|v1.2.3|') +@example('|v99.99.99|') +@example(f'|{DEFAULT_VERSION_STRING}|') +@given(strategies.from_regex(DEVDOC_VERSION_PATTERN, fullmatch=True)) +@settings(max_examples=5000) +def test_devdoc_regex_pattern(full_match): + + # Not empty + assert full_match, 'Devdoc regex pattern matched an empty value: "{version_string}"' + + # Anchors + assert full_match.startswith('|'), 'Version string does not end in "|" delimiter: "{version_string}"' + assert full_match.endswith('|'), 'Version string does not end in "|" delimiter: "{version_string}"' + + # Max Size + numbers_only = re.sub("[^0-9]", "", full_match) + # I mean really... who has a version with more than 6 numbers (v99.99.99) + assert len(numbers_only) <= 10, 'Version string is too long: "{version_string}"' + + # "v" specifier + version_string = full_match[1:-1] + assert version_string.startswith('v'), 'Version string does not start with "v": "{version_string}"' + assert version_string.count('v') == 1, 'Version string contains more than one "v": "{version_string}"' + + # Version parts + assert version_string.count('.') == 2, f'Version string has more than two periods: "{version_string}"' + parts = version_string[1:] + version_parts = parts.split('.') + assert len(version_parts) == 3, f'Version string has more than three parts: "{version_string}"' + + # Parts are numbers + assert all(p.isdigit() for p in version_parts), f'Non-digit found in version string: "{version_string}"' diff --git a/tests/unit/test_prometheus.py b/tests/unit/test_prometheus.py index 227f54520..02428df44 100644 --- a/tests/unit/test_prometheus.py +++ b/tests/unit/test_prometheus.py @@ -274,3 +274,7 @@ class TestGenerateJSON(unittest.TestCase): 123.000456, "exemplar": {}}, {"sample_name": "ts", "labels": {"foo": "f"}, "value": "0.0", "timestamp": 123.000000456, "exemplar": {}}], "help": "help", "type": "unknown"}}"""), json.loads( self.json_exporter.generate_latest_json())) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/utils/blockchain.py b/tests/utils/blockchain.py index 34b8b38e5..69f9d0641 100644 --- a/tests/utils/blockchain.py +++ b/tests/utils/blockchain.py @@ -15,20 +15,23 @@ You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ +import os +from pathlib import Path +from typing import List, Tuple, Union import maya -import os from eth_tester.exceptions import TransactionFailed from eth_utils import to_canonical_address from hexbytes import HexBytes -from typing import List, Tuple, Union from web3 import Web3 from nucypher.blockchain.economics import BaseEconomics, StandardTokenEconomics from nucypher.blockchain.eth.actors import ContractAdministrator from nucypher.blockchain.eth.interfaces import BlockchainDeployerInterface, BlockchainInterfaceFactory from nucypher.blockchain.eth.registry import InMemoryContractRegistry -from nucypher.blockchain.eth.sol.compile import SolidityCompiler +from nucypher.blockchain.eth.sol.compile.compile import multiversion_compile +from nucypher.blockchain.eth.sol.compile.constants import TEST_SOLIDITY_SOURCE_ROOT, SOLIDITY_SOURCE_ROOT +from nucypher.blockchain.eth.sol.compile.types import SourceBundle from nucypher.blockchain.eth.token import NU from nucypher.blockchain.eth.utils import epoch_to_period from nucypher.crypto.powers import TransactingPower @@ -40,8 +43,7 @@ from tests.constants import ( INSECURE_DEVELOPMENT_PASSWORD, NUMBER_OF_ETH_TEST_ACCOUNTS, NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS, - NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS, - TEST_CONTRACTS_DIR + NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS, PYEVM_DEV_URI ) @@ -72,56 +74,50 @@ class TesterBlockchain(BlockchainDeployerInterface): Blockchain subclass with additional test utility methods and options. """ - __test__ = False # prohibit Pytest from picking it up + __test__ = False # prohibit pytest from collecting this object as a test - _instance = None + # Solidity + SOURCES: List[SourceBundle] = [ + SourceBundle(base_path=SOLIDITY_SOURCE_ROOT, + other_paths=(TEST_SOLIDITY_SOURCE_ROOT,)) + ] - GAS_STRATEGIES = {**BlockchainDeployerInterface.GAS_STRATEGIES, - 'free': free_gas_price_strategy} + # Web3 + GAS_STRATEGIES = {**BlockchainDeployerInterface.GAS_STRATEGIES, 'free': free_gas_price_strategy} + PROVIDER_URI = PYEVM_DEV_URI DEFAULT_GAS_STRATEGY = 'free' - _PROVIDER_URI = 'tester://pyevm' - _compiler = SolidityCompiler(source_dirs=[(SolidityCompiler.default_contract_dir(), {TEST_CONTRACTS_DIR})]) - _test_account_cache = list() - - _default_test_accounts = NUMBER_OF_ETH_TEST_ACCOUNTS - # Reserved addresses _ETHERBASE = 0 _ALICE = 1 _BOB = 2 _FIRST_STAKER = 5 - _stakers_range = range(NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS) _FIRST_URSULA = _FIRST_STAKER + NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS - _ursulas_range = range(NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS) - _default_token_economics = StandardTokenEconomics() + # Internal + __STAKERS_RANGE = range(NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS) + __WORKERS_RANGE = range(NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS) + __ACCOUNT_CACHE = list() + + # Defaults + DEFAULT_ECONOMICS = StandardTokenEconomics() def __init__(self, - test_accounts=None, - poa=True, - light=False, - eth_airdrop=False, - free_transactions=False, - compiler: SolidityCompiler = None, - mock_backend: bool = False, + test_accounts: int = NUMBER_OF_ETH_TEST_ACCOUNTS, + poa: bool = True, + light: bool = False, + eth_airdrop: bool = False, + free_transactions: bool = False, *args, **kwargs): - if not test_accounts: - test_accounts = self._default_test_accounts self.free_transactions = free_transactions EXPECTED_CONFIRMATION_TIME_IN_SECONDS['free'] = 5 # Just some upper-limit - if compiler: - TesterBlockchain._compiler = compiler - - super().__init__(provider_uri=self._PROVIDER_URI, + super().__init__(provider_uri=self.PROVIDER_URI, provider_process=None, poa=poa, light=light, - compiler=self._compiler, - dry_run=mock_backend, *args, **kwargs) self.log = Logger("test-blockchain") @@ -138,6 +134,17 @@ class TesterBlockchain(BlockchainDeployerInterface): if eth_airdrop is True: # ETH for everyone! self.ether_airdrop(amount=DEVELOPMENT_ETH_AIRDROP_AMOUNT) + # TODO: DRY this up + def connect(self, compile_now: bool = True, ignore_solidity_check: bool = False) -> bool: + super().connect() + if compile_now: + # Execute the compilation if we're recompiling + # Otherwise read compiled contract data from the registry. + check = not ignore_solidity_check + compiled_contracts = multiversion_compile(source_bundles=self.SOURCES, compiler_version_check=check) + self._raw_contract_cache = compiled_contracts + return self.is_connected + def attach_middleware(self): if self.free_transactions: self.w3.eth.setGasPriceStrategy(free_gas_price_strategy) @@ -160,7 +167,7 @@ class TesterBlockchain(BlockchainDeployerInterface): for _ in range(quantity): address = self.provider.ethereum_tester.add_account('0x' + os.urandom(32).hex()) addresses.append(address) - self._test_account_cache.append(address) + self.__ACCOUNT_CACHE.append(address) self.log.info('Generated new insecure account {}'.format(address)) return addresses @@ -192,8 +199,8 @@ class TesterBlockchain(BlockchainDeployerInterface): raise ValueError("Specify hours, seconds, or periods, not a combination") if periods: - duration = self._default_token_economics.seconds_per_period * periods - base = self._default_token_economics.seconds_per_period + duration = self.DEFAULT_ECONOMICS.seconds_per_period * periods + base = self.DEFAULT_ECONOMICS.seconds_per_period elif hours: duration = hours * (60*60) base = 60 * 60 @@ -211,7 +218,7 @@ class TesterBlockchain(BlockchainDeployerInterface): delta = maya.timedelta(seconds=end_timestamp-now) self.log.info(f"Time traveled {delta} " - f"| period {epoch_to_period(epoch=end_timestamp, seconds_per_period=self._default_token_economics.seconds_per_period)} " + f"| period {epoch_to_period(epoch=end_timestamp, seconds_per_period=self.DEFAULT_ECONOMICS.seconds_per_period)} " f"| epoch {end_timestamp}") @classmethod @@ -219,7 +226,7 @@ class TesterBlockchain(BlockchainDeployerInterface): """For use with metric testing scripts""" registry = InMemoryContractRegistry() - testerchain = cls(compiler=SolidityCompiler()) + testerchain = cls() BlockchainInterfaceFactory.register_interface(testerchain) power = TransactingPower(password=INSECURE_DEVELOPMENT_PASSWORD, account=testerchain.etherbase_account) @@ -229,7 +236,7 @@ class TesterBlockchain(BlockchainDeployerInterface): origin = testerchain.client.etherbase deployer = ContractAdministrator(deployer_address=origin, registry=registry, - economics=economics or cls._default_token_economics, + economics=economics or cls.DEFAULT_ECONOMICS, staking_escrow_test_mode=True) _receipts = deployer.deploy_network_contracts(interactive=False) @@ -248,22 +255,22 @@ class TesterBlockchain(BlockchainDeployerInterface): return self.client.accounts[self._BOB] def ursula_account(self, index): - if index not in self._ursulas_range: + if index not in self.__WORKERS_RANGE: raise ValueError(f"Ursula index must be lower than {NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS}") return self.client.accounts[index + self._FIRST_URSULA] def staker_account(self, index): - if index not in self._stakers_range: + if index not in self.__STAKERS_RANGE: raise ValueError(f"Staker index must be lower than {NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS}") return self.client.accounts[index + self._FIRST_STAKER] @property def ursulas_accounts(self): - return list(self.ursula_account(i) for i in self._ursulas_range) + return list(self.ursula_account(i) for i in self.__WORKERS_RANGE) @property def stakers_accounts(self): - return list(self.staker_account(i) for i in self._stakers_range) + return list(self.staker_account(i) for i in self.__STAKERS_RANGE) @property def unassigned_accounts(self): diff --git a/tests/utils/ursula.py b/tests/utils/ursula.py index 0049342cf..2010a4b46 100644 --- a/tests/utils/ursula.py +++ b/tests/utils/ursula.py @@ -23,15 +23,23 @@ import tempfile from cryptography.x509 import Certificate from typing import Iterable, List, Optional, Set +from nucypher.characters.lawful import Bob +from nucypher.crypto.utils import canonical_address_from_umbral_key + from nucypher.blockchain.eth.actors import Staker from nucypher.blockchain.eth.interfaces import BlockchainInterface from nucypher.characters.lawful import Ursula from nucypher.config.characters import UrsulaConfiguration from nucypher.crypto.powers import TransactingPower +from nucypher.policy.collections import WorkOrder, IndisputableEvidence from tests.constants import ( MOCK_URSULA_DB_FILEPATH, NUMBER_OF_URSULAS_IN_DEVELOPMENT_NETWORK ) +from umbral import pre +from umbral.curvebn import CurveBN +from umbral.keys import UmbralPrivateKey +from umbral.signing import Signer def select_test_port() -> int: @@ -166,3 +174,51 @@ def start_pytest_ursula_services(ursula: Ursula) -> Certificate: MOCK_KNOWN_URSULAS_CACHE = dict() MOCK_URSULA_STARTING_PORT = 51000 # select_test_port() + + +def _mock_ursula_reencrypts(ursula, corrupt_cfrag: bool = False): + delegating_privkey = UmbralPrivateKey.gen_key() + _symmetric_key, capsule = pre._encapsulate(delegating_privkey.get_pubkey()) + signing_privkey = UmbralPrivateKey.gen_key() + signing_pubkey = signing_privkey.get_pubkey() + signer = Signer(signing_privkey) + priv_key_bob = UmbralPrivateKey.gen_key() + pub_key_bob = priv_key_bob.get_pubkey() + kfrags = pre.generate_kfrags(delegating_privkey=delegating_privkey, + signer=signer, + receiving_pubkey=pub_key_bob, + threshold=2, + N=4, + sign_delegating_key=False, + sign_receiving_key=False) + capsule.set_correctness_keys(delegating_privkey.get_pubkey(), pub_key_bob, signing_pubkey) + + ursula_pubkey = ursula.stamp.as_umbral_pubkey() + + alice_address = canonical_address_from_umbral_key(signing_pubkey) + blockhash = bytes(32) + + specification = b''.join((bytes(capsule), + bytes(ursula_pubkey), + bytes(ursula.decentralized_identity_evidence), + alice_address, + blockhash)) + + bobs_signer = Signer(priv_key_bob) + task_signature = bytes(bobs_signer(specification)) + + metadata = bytes(ursula.stamp(task_signature)) + + cfrag = pre.reencrypt(kfrags[0], capsule, metadata=metadata) + + if corrupt_cfrag: + cfrag.proof.bn_sig = CurveBN.gen_rand(capsule.params.curve) + + cfrag_signature = ursula.stamp(bytes(cfrag)) + + bob = Bob.from_public_keys(verifying_key=pub_key_bob) + task = WorkOrder.PRETask(capsule, task_signature, cfrag, cfrag_signature) + work_order = WorkOrder(bob, None, alice_address, {capsule: task}, None, ursula, blockhash) + + evidence = IndisputableEvidence(task, work_order) + return evidence