From b8949a7972d6d63f465ff598770942a685701853 Mon Sep 17 00:00:00 2001 From: Kieran Prasch Date: Fri, 19 Jun 2020 16:24:49 -0700 Subject: [PATCH] Follow the yellow brick road --- nucypher/blockchain/eth/interfaces.py | 11 +++--- .../blockchain/eth/sol/compile/aggregation.py | 26 ++++++++++++-- .../blockchain/eth/sol/compile/collect.py | 23 ++++++------ .../blockchain/eth/sol/compile/compile.py | 24 +++++++------ nucypher/blockchain/eth/sol/compile/config.py | 35 +++++-------------- .../blockchain/eth/sol/compile/constants.py | 30 +++------------- nucypher/blockchain/eth/sol/compile/solc.py | 8 ++--- nucypher/blockchain/eth/sol/compile/types.py | 10 ++++-- .../test_handle_multiversion_contracts.py | 14 ++++---- .../test_contracts_upgradeability.py | 6 ++-- 10 files changed, 90 insertions(+), 97 deletions(-) diff --git a/nucypher/blockchain/eth/interfaces.py b/nucypher/blockchain/eth/interfaces.py index 6745a1c33..5ed6cc045 100644 --- a/nucypher/blockchain/eth/interfaces.py +++ b/nucypher/blockchain/eth/interfaces.py @@ -19,7 +19,6 @@ import os import pprint import threading import time -from pathlib import Path from typing import Callable, Tuple, Union, NamedTuple from urllib.parse import urlparse @@ -808,9 +807,10 @@ class BlockchainDeployerInterface(BlockchainInterface): TIMEOUT = 600 # seconds _CONTRACT_FACTORY = VersionedContract + # TODO: Make more func - use as a parameter # Source directories to (recursively) compile - SOURCES: Tuple[Path, ...] = [ - SOLIDITY_SOURCE_ROOT + SOURCES: Tuple[SourceBundle, ...] = [ + SourceBundle(source_dirs=(SOLIDITY_SOURCE_ROOT, ), import_root=SOLIDITY_SOURCE_ROOT) ] _raw_contract_cache = NO_COMPILATION_PERFORMED @@ -821,13 +821,14 @@ class BlockchainDeployerInterface(BlockchainInterface): class DeploymentFailed(RuntimeError): pass + def connect(self, compile_now: bool = True, ignore_solidity_check: bool = False) -> bool: super().connect() if compile_now: # Execute the compilation if we're recompiling # Otherwise read compiled contract data from the registry. - compiled_contracts = multiversion_compile(compiler_version_check=not ignore_solidity_check, - solidity_source_dirs=self.SOURCES) + check = not ignore_solidity_check + compiled_contracts = multiversion_compile(source_bundles=self.SOURCES, compiler_version_check=check) self._raw_contract_cache = compiled_contracts return self.is_connected diff --git a/nucypher/blockchain/eth/sol/compile/aggregation.py b/nucypher/blockchain/eth/sol/compile/aggregation.py index 1779ed6db..6168c7a06 100644 --- a/nucypher/blockchain/eth/sol/compile/aggregation.py +++ b/nucypher/blockchain/eth/sol/compile/aggregation.py @@ -16,13 +16,33 @@ along with nucypher. If not, see . """ -from cytoolz.dicttoolz import merge +import re +from re import Pattern from typing import Dict -from nucypher.blockchain.eth.sol.compile.constants import DEFAULT_VERSION_STRING, DEVDOC_VERSION_PATTERN, SOLC_LOGGER +from cytoolz.dicttoolz import merge + +from nucypher.blockchain.eth.sol.compile.constants import DEFAULT_VERSION_STRING, SOLC_LOGGER from nucypher.blockchain.eth.sol.compile.exceptions import CompilationError, ProgrammingError from nucypher.blockchain.eth.sol.compile.types import VersionedContractOutputs, CompiledContractOutputs +# RE pattern for matching solidity source compile version specification in devdoc details. +DEVDOC_VERSION_PATTERN: Pattern = re.compile(r""" +\A # Anchor must be first +\| # Anchored pipe literal at beginning of version definition +( # Start Inner capture group +v # Capture version starting from symbol v +\d+ # At least one digit of major version +\. # Digits splitter +\d+ # At least one digit of minor version +\. # Digits splitter +\d+ # At least one digit of patch +) # End of capturing +\| # Anchored end of version definition | +\Z # Anchor must be the end of the match +""", re.VERBOSE) + + def extract_version(compiled_contract_outputs: dict) -> str: """ @@ -91,7 +111,7 @@ def validate_merge(existing_version: CompiledContractOutputs, def merge_contract_sources(*compiled_sources): - return merge(*compiled_sources) # TODO: Handle file-lecel output aggregation + return merge(*compiled_sources) # TODO: Handle file-level output aggregation def merge_contract_outputs(*compiled_versions) -> VersionedContractOutputs: diff --git a/nucypher/blockchain/eth/sol/compile/collect.py b/nucypher/blockchain/eth/sol/compile/collect.py index 2bfba1a76..6e38ad9ce 100644 --- a/nucypher/blockchain/eth/sol/compile/collect.py +++ b/nucypher/blockchain/eth/sol/compile/collect.py @@ -16,6 +16,7 @@ along with nucypher. If not, see . """ +from nucypher.blockchain.eth.sol.compile.types import SourceBundle from nucypher.exceptions import DevelopmentInstallationRequired try: import tests @@ -34,20 +35,22 @@ def source_filter(filename: str) -> bool: return is_solidity_file and not contains_ignored_prefix -def collect_sources(source_dir: Path) -> Dict[str, Dict[str, List[str]]]: +def collect_sources(source_bundle: SourceBundle) -> Dict[str, Dict[str, List[str]]]: """ Returns a compiler-ready mapping of solidity source files in source_dir (recursive) Walks source_dir top-down to the bottom filepath of each subdirectory recursively and filtrates by __source_filter, setting values into `source_paths`. """ source_paths: Dict[str, Dict[str, List[str]]] = dict() - source_walker: Iterator = os.walk(top=str(source_dir), topdown=True) - # Collect single directory - for root, dirs, files in source_walker: - # Collect files in source dir - for filename in filter(source_filter, files): - path = Path(root) / filename - source_paths[filename] = dict(urls=[str(path.resolve(strict=True))]) - SOLC_LOGGER.debug(f"Collecting solidity source {path}") - SOLC_LOGGER.info(f"Collected {len(source_paths)} solidity source files at {source_dir}") + for source_dir in source_bundle.source_dirs: + source_walker: Iterator = list(os.walk(top=str(source_dir.resolve(strict=True)), topdown=True)) # TODO: Remove list caster + # Collect single directory + for root, dirs, files in source_walker: + # Collect files in source dir + mapped_path = str(root).replace(str(source_bundle), '') + for filename in filter(source_filter, files): + path = Path(root) / filename + source_paths[filename] = dict(urls=[mapped_path]) + SOLC_LOGGER.debug(f"Collecting solidity source {path}") + SOLC_LOGGER.info(f"Collected {len(source_paths)} solidity source files at {source_bundle}") return source_paths diff --git a/nucypher/blockchain/eth/sol/compile/compile.py b/nucypher/blockchain/eth/sol/compile/compile.py index 4a6cbef30..44ba91c08 100644 --- a/nucypher/blockchain/eth/sol/compile/compile.py +++ b/nucypher/blockchain/eth/sol/compile/compile.py @@ -17,37 +17,39 @@ along with nucypher. If not, see . import itertools - +from pathlib import Path from typing import Tuple, List, Dict from cytoolz.dicttoolz import merge, merge_with -from pathlib import Path from nucypher.blockchain.eth.sol.__conf__ import SOLIDITY_COMPILER_VERSION from nucypher.blockchain.eth.sol.compile.aggregation import merge_contract_outputs from nucypher.blockchain.eth.sol.compile.collect import collect_sources from nucypher.blockchain.eth.sol.compile.config import BASE_COMPILER_CONFIGURATION -from nucypher.blockchain.eth.sol.compile.config import ALLOWED_PATHS from nucypher.blockchain.eth.sol.compile.solc import __execute -from nucypher.blockchain.eth.sol.compile.types import VersionString, VersionedContractOutputs, CompiledContractOutputs +from nucypher.blockchain.eth.sol.compile.types import ( + VersionString, + VersionedContractOutputs, + CompiledContractOutputs, + SourceBundle +) -def compile_sources(solidity_source_dir: Path, version_check: bool = True) -> Dict: +def compile_sources(source_bundle: SourceBundle, version_check: bool = True) -> Dict: """Compiled solidity contracts for a single source directory""" - sources = collect_sources(source_dir=solidity_source_dir) + sources = collect_sources(source_bundle=source_bundle) solc_configuration = merge(BASE_COMPILER_CONFIGURATION, dict(sources=sources)) # do not mutate - allowed_paths = ','.join(list(set(str(p) for p in ALLOWED_PATHS))) # unique ignore_version_check: bool = not version_check version: VersionString = VersionString(SOLIDITY_COMPILER_VERSION) if ignore_version_check else None - compiler_output = __execute(compiler_version=version, input_config=solc_configuration, allowed_paths=allowed_paths) + compiler_output = __execute(compiler_version=version, input_config=solc_configuration, base_path=str(source_bundle.import_root)) return compiler_output -def multiversion_compile(solidity_source_dirs: Tuple[Path, ...], compiler_version_check: bool = True) -> VersionedContractOutputs: +def multiversion_compile(source_bundles: Tuple[SourceBundle, ...], compiler_version_check: bool = True) -> VersionedContractOutputs: """Compile contracts from `source_dirs` and aggregate the resulting source contract outputs by version""" raw_compiler_results: List[CompiledContractOutputs] = list() - for source_dir in solidity_source_dirs: - compile_result = compile_sources(solidity_source_dir=source_dir, version_check=compiler_version_check) + for bundle in source_bundles: + compile_result = compile_sources(source_bundle=bundle, version_check=compiler_version_check) raw_compiler_results.append(compile_result['contracts']) raw_compiled_contracts = itertools.chain.from_iterable(output.values() for output in raw_compiler_results) versioned_contract_outputs = merge_with(merge_contract_outputs, *raw_compiled_contracts) diff --git a/nucypher/blockchain/eth/sol/compile/config.py b/nucypher/blockchain/eth/sol/compile/config.py index 8ef8dac71..c6cf62ee2 100644 --- a/nucypher/blockchain/eth/sol/compile/config.py +++ b/nucypher/blockchain/eth/sol/compile/config.py @@ -18,8 +18,6 @@ along with nucypher. If not, see . from typing import List, Dict -from nucypher.blockchain.eth.sol.compile.constants import NUCYPHER_CONTRACTS_DIR, ZEPPELIN_DIR, ARAGON_DIR, ARAGON, ZEPPELIN, \ - SOLIDITY_SOURCE_ROOT, TEST_SOLIDITY_SOURCE_ROOT from nucypher.blockchain.eth.sol.compile.types import CompilerConfiguration @@ -33,23 +31,15 @@ WARNING: Do not change these values unless you know what you are doing. """ +# Debug +# ----- # How to treat revert (and require) reason strings. # "default", "strip", "debug" and "verboseDebug". # "default" does not inject compiler-generated revert strings and keeps user-supplied ones. # "strip" removes all revert strings (if possible, i.e. if literals are used) keeping side-effects # "debug" injects strings for compiler-generated internal reverts, implemented for ABI encoders V1 and V2 for now. # "verboseDebug" even appends further information to user-supplied revert strings (not yet implemented) -DEBUG = 'default' - -# Hardcoded for added sanity. -# New top-level contract source directories must be listed here. -# Paths can be commented out to prevent default permission. -# In tests, this list can be mutated to temporarily allow compilation -# of source files that are typically not permitted. -ALLOWED_PATHS = [ - SOLIDITY_SOURCE_ROOT, - TEST_SOLIDITY_SOURCE_ROOT -] +# DEBUG = 'default' # Source code language. Currently supported are "Solidity" and "Yul". LANGUAGE: str = 'Solidity' @@ -59,7 +49,7 @@ EVM_VERSION: str = 'berlin' # File level compiler outputs (needs empty string as contract name): FILE_OUTPUTS: List[str] = [ - # 'ast' # AST of all source files # TODO: Handle AST for static analysis + 'ast' # AST of all source files # 'legacyAST' # legacy AST of all source files ] @@ -88,10 +78,8 @@ CONTRACT_OUTPUTS: List[str] = [ # 'ewasm.wasm', # eWASM binary format (not supported at the moment) ] - -# Optional -# Switch optimizer components on or off in detail. -# The "enabled" switch above provides two defaults which can be tweaked here. +# Optimizer Details - Switch optimizer components on or off in detail. +# The "enabled" switch above provides two defaults which can be tweaked here (yul, and ...). OPTIMIZER_DETAILS = dict( peephole=True, # The peephole optimizer is always on if no details are given (switch it off here). jumpdestRemover=True, # The unused jumpdest remover is always on if no details are given (switch it off here). @@ -103,7 +91,7 @@ OPTIMIZER_DETAILS = dict( # The new Yul optimizer. Mostly operates on the code of ABIEncoderV2 and inline assembly. # It is activated together with the global optimizer setting and can be deactivated here. # Before Solidity 0.6.0 it had to be activated through this switch. Also see 'yulDetails options'. - yul=False + yul=True ) # Optimize for how many times you intend to run the code. @@ -117,19 +105,14 @@ OPTIMIZER_SETTINGS = dict( # details=OPTIMIZER_DETAILS # Optional - If "details" is given, "enabled" can be omitted. ) -IMPORT_REMAPPING: List[str] = [ - f"contracts={NUCYPHER_CONTRACTS_DIR.resolve()}", - f"{ZEPPELIN}={ZEPPELIN_DIR.resolve()}", - f"{ARAGON}={ARAGON_DIR.resolve()}", -] - +# Complete compiler settings COMPILER_SETTINGS: Dict = dict( - remappings=IMPORT_REMAPPING, optimizer=OPTIMIZER_SETTINGS, evmVersion=EVM_VERSION, outputSelection={"*": {"*": CONTRACT_OUTPUTS, "": FILE_OUTPUTS}} # all contacts(*), all files("") ) +# Base configuration for programmatic usage BASE_COMPILER_CONFIGURATION = CompilerConfiguration( language=LANGUAGE, settings=COMPILER_SETTINGS, diff --git a/nucypher/blockchain/eth/sol/compile/constants.py b/nucypher/blockchain/eth/sol/compile/constants.py index 54318a129..ab8ceb5b3 100644 --- a/nucypher/blockchain/eth/sol/compile/constants.py +++ b/nucypher/blockchain/eth/sol/compile/constants.py @@ -19,29 +19,24 @@ along with nucypher. If not, see . import re from logging import Logger from pathlib import Path -from typing import Tuple, Pattern +from typing import Tuple, Pattern, NamedTuple, List from nucypher.config.constants import NUCYPHER_TEST_DIR + + + # Logging SOLC_LOGGER = Logger("solidity-compilation") # Vocabulary CONTRACTS = 'contracts' -ZEPPELIN: str = 'zeppelin' -ARAGON: str = 'aragon' - TEST_SOLIDITY_SOURCE_ROOT: Path = Path(NUCYPHER_TEST_DIR).parent / CONTRACTS / CONTRACTS from nucypher.blockchain.eth import sol SOLIDITY_SOURCE_ROOT: Path = Path(sol.__file__).parent / 'source' -# Import Remapping -ZEPPELIN_DIR: Path = SOLIDITY_SOURCE_ROOT / ZEPPELIN -ARAGON_DIR: Path = SOLIDITY_SOURCE_ROOT / ARAGON -NUCYPHER_CONTRACTS_DIR: Path = SOLIDITY_SOURCE_ROOT / 'contracts' - # Do not compile contracts containing... IGNORE_CONTRACT_PREFIXES: Tuple[str, ...] = ( 'Abstract', @@ -50,19 +45,4 @@ IGNORE_CONTRACT_PREFIXES: Tuple[str, ...] = ( DEFAULT_VERSION_STRING: str = 'v0.0.0' # for both compiler and devdoc versions (must fully match regex pattern below) - -# RE pattern for matching solidity source compile version specification in devdoc details. -DEVDOC_VERSION_PATTERN: Pattern = re.compile(r""" -\A # Anchor must be first -\| # Anchored pipe literal at beginning of version definition -( # Start Inner capture group -v # Capture version starting from symbol v -\d+ # At least one digit of major version -\. # Digits splitter -\d+ # At least one digit of minor version -\. # Digits splitter -\d+ # At least one digit of patch -) # End of capturing -\| # Anchored end of version definition | -\Z # Anchor must be the end of the match -""", re.VERBOSE) +# TODO: Move closer to usage diff --git a/nucypher/blockchain/eth/sol/compile/solc.py b/nucypher/blockchain/eth/sol/compile/solc.py index f2b20cd4e..616f73322 100644 --- a/nucypher/blockchain/eth/sol/compile/solc.py +++ b/nucypher/blockchain/eth/sol/compile/solc.py @@ -20,14 +20,14 @@ from logging import Logger from typing import Dict -from nucypher.blockchain.eth.sol.compile.config import IMPORT_REMAPPING, OPTIMIZER_RUNS, ALLOWED_PATHS +from nucypher.blockchain.eth.sol.compile.config import OPTIMIZER_RUNS from nucypher.blockchain.eth.sol.compile.constants import SOLC_LOGGER from nucypher.blockchain.eth.sol.compile.exceptions import CompilationError from nucypher.blockchain.eth.sol.compile.types import VersionString from nucypher.exceptions import DevelopmentInstallationRequired -def __execute(compiler_version: VersionString, input_config: Dict, allowed_paths: str): +def __execute(compiler_version: VersionString, input_config: Dict, base_path: str): """Executes the solcx command and underlying solc wrapper""" log = Logger('execute-solcx') @@ -40,11 +40,11 @@ def __execute(compiler_version: VersionString, input_config: Dict, allowed_paths # Prepare Solc Command solc_binary_path: str = get_executable(version=compiler_version) - SOLC_LOGGER.info(f"Compiling with import remappings {' '.join(IMPORT_REMAPPING)} and allowed paths {ALLOWED_PATHS}") + SOLC_LOGGER.info(f"Compiling with base path") # TODO: Add base path # Execute Compilation try: - compiler_output = compile_standard(input_data=input_config, allow_paths=allowed_paths) + compiler_output = compile_standard(input_data=input_config, base_path=base_path) except FileNotFoundError: raise CompilationError("The solidity compiler is not at the specified path. " "Check that the file exists and is executable.") diff --git a/nucypher/blockchain/eth/sol/compile/types.py b/nucypher/blockchain/eth/sol/compile/types.py index fe5859766..40a2db965 100644 --- a/nucypher/blockchain/eth/sol/compile/types.py +++ b/nucypher/blockchain/eth/sol/compile/types.py @@ -14,9 +14,8 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with nucypher. If not, see . """ - - -from typing import Dict, List, Union, NewType +from pathlib import Path +from typing import Dict, List, Union, NewType, NamedTuple, Tuple, Optional class ABI(Dict): @@ -42,3 +41,8 @@ class CompilerConfiguration(Dict): language: str sources: Dict[str, Dict[str, str]] settings: Dict + + +class SourceBundle(NamedTuple): + source_dirs: Tuple[Path] + import_root: Optional[Path] = None diff --git a/tests/acceptance/blockchain/interfaces/test_handle_multiversion_contracts.py b/tests/acceptance/blockchain/interfaces/test_handle_multiversion_contracts.py index 9e3c30eaf..6617e926b 100644 --- a/tests/acceptance/blockchain/interfaces/test_handle_multiversion_contracts.py +++ b/tests/acceptance/blockchain/interfaces/test_handle_multiversion_contracts.py @@ -17,8 +17,7 @@ from pathlib import Path -from nucypher.blockchain.eth.sol.compile.config import ALLOWED_PATHS - +from nucypher.blockchain.eth.sol.compile.types import SourceBundle from nucypher.blockchain.eth.interfaces import BlockchainDeployerInterface, BlockchainInterfaceFactory from nucypher.blockchain.eth.registry import InMemoryContractRegistry from nucypher.crypto.powers import TransactingPower @@ -29,13 +28,14 @@ def test_deployer_interface_multiversion_contract(): # Prepare compiler base_dir = Path(__file__).parent / 'contracts' / 'multiversion' - v1_dir, v2_dir = base_dir / 'v1', base_dir / 'v2' + v1_dir, v2_dir = Path(base_dir / 'v1'), Path(base_dir / 'v2') + source_dirs = v1_dir, v2_dir # I am a contract administrator and I an compiling a new updated version of an existing contract... - # Represents "Manually hardcoding" a new permitted compile path in compile.py - # and new source directory on BlockchainDeployerInterface.SOURCES. - ALLOWED_PATHS.append(base_dir) - BlockchainDeployerInterface.SOURCES = (v1_dir, v2_dir) + # Represents "Manually hardcoding" a new source directory on BlockchainDeployerInterface.SOURCES. + BlockchainDeployerInterface.SOURCES = ( + SourceBundle(source_dirs=source_dirs, import_root=base_dir), + ) # Prepare chain BlockchainInterfaceFactory._interfaces.clear() diff --git a/tests/contracts/test_contracts_upgradeability.py b/tests/contracts/test_contracts_upgradeability.py index 12863952c..3afbe27b0 100644 --- a/tests/contracts/test_contracts_upgradeability.py +++ b/tests/contracts/test_contracts_upgradeability.py @@ -93,13 +93,13 @@ def deploy_earliest_contract(blockchain_interface: BlockchainDeployerInterface, pass # Skip errors related to initialization -@pytest.mark.skip('GH 403') # FIXME +# FIXME: Needs Completion def test_upgradeability(temp_dir_path, token_economics): # Prepare remote source for compilation download_github_dir(GITHUB_SOURCE_LINK, temp_dir_path) # Prepare the blockchain - provider_uri = 'tester://pyevm/2' + provider_uri = 'tester://pyevm/2' # TODO: Testerchain caching Issues try: blockchain_interface = BlockchainDeployerInterface(provider_uri=provider_uri, gas_strategy=free_gas_price_strategy) @@ -143,6 +143,6 @@ def test_upgradeability(temp_dir_path, token_economics): adjudicator_deployer.upgrade(contract_version="latest", confirmations=0) finally: - # Unregister interface + # Unregister interface # TODO: Move to method? with contextlib.suppress(KeyError): del BlockchainInterfaceFactory._interfaces[provider_uri]