Nucypher
/
nucypher
mirror of https://github.com/nucypher/nucypher.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Follow the yellow brick road

pull/2439/head
Kieran Prasch 2020-06-19 16:24:49 -07:00 committed by vzotova
parent 0dec898159
commit b8949a7972
10 changed files with 90 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
nucypher/blockchain/eth/interfaces.py
View File

@ -19,7 +19,6 @@ import os
import pprint
import threading
import time
from pathlib import Path
from typing import Callable, Tuple, Union, NamedTuple
from urllib.parse import urlparse
@ -808,9 +807,10 @@ class BlockchainDeployerInterface(BlockchainInterface):
TIMEOUT = 600 # seconds
_CONTRACT_FACTORY = VersionedContract
# TODO: Make more func - use as a parameter
# Source directories to (recursively) compile
SOURCES: Tuple[Path, ...] = [
SOLIDITY_SOURCE_ROOT
SOURCES: Tuple[SourceBundle, ...] = [
SourceBundle(source_dirs=(SOLIDITY_SOURCE_ROOT, ), import_root=SOLIDITY_SOURCE_ROOT)
]
_raw_contract_cache = NO_COMPILATION_PERFORMED
@ -821,13 +821,14 @@ class BlockchainDeployerInterface(BlockchainInterface):
class DeploymentFailed(RuntimeError):
pass
def connect(self, compile_now: bool = True, ignore_solidity_check: bool = False) -> bool:
super().connect()
if compile_now:
# Execute the compilation if we're recompiling
# Otherwise read compiled contract data from the registry.
compiled_contracts = multiversion_compile(compiler_version_check=not ignore_solidity_check,
solidity_source_dirs=self.SOURCES)
check = not ignore_solidity_check
compiled_contracts = multiversion_compile(source_bundles=self.SOURCES, compiler_version_check=check)
self._raw_contract_cache = compiled_contracts
return self.is_connected

26
nucypher/blockchain/eth/sol/compile/aggregation.py
View File

@ -16,13 +16,33 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from cytoolz.dicttoolz import merge
import re
from re import Pattern
from typing import Dict
from nucypher.blockchain.eth.sol.compile.constants import DEFAULT_VERSION_STRING, DEVDOC_VERSION_PATTERN, SOLC_LOGGER
from cytoolz.dicttoolz import merge
from nucypher.blockchain.eth.sol.compile.constants import DEFAULT_VERSION_STRING, SOLC_LOGGER
from nucypher.blockchain.eth.sol.compile.exceptions import CompilationError, ProgrammingError
from nucypher.blockchain.eth.sol.compile.types import VersionedContractOutputs, CompiledContractOutputs
# RE pattern for matching solidity source compile version specification in devdoc details.
DEVDOC_VERSION_PATTERN: Pattern = re.compile(r"""
\A # Anchor must be first
\| # Anchored pipe literal at beginning of version definition
( # Start Inner capture group
v # Capture version starting from symbol v
\d+ # At least one digit of major version
\. # Digits splitter
\d+ # At least one digit of minor version
\. # Digits splitter
\d+ # At least one digit of patch
) # End of capturing
\| # Anchored end of version definition |
\Z # Anchor must be the end of the match
""", re.VERBOSE)
def extract_version(compiled_contract_outputs: dict) -> str:
"""
@ -91,7 +111,7 @@ def validate_merge(existing_version: CompiledContractOutputs,
def merge_contract_sources(*compiled_sources):
return merge(*compiled_sources) # TODO: Handle file-lecel output aggregation
return merge(*compiled_sources) # TODO: Handle file-level output aggregation
def merge_contract_outputs(*compiled_versions) -> VersionedContractOutputs:

23
nucypher/blockchain/eth/sol/compile/collect.py
View File

@ -16,6 +16,7 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from nucypher.blockchain.eth.sol.compile.types import SourceBundle
from nucypher.exceptions import DevelopmentInstallationRequired
try:
import tests
@ -34,20 +35,22 @@ def source_filter(filename: str) -> bool:
return is_solidity_file and not contains_ignored_prefix
def collect_sources(source_dir: Path) -> Dict[str, Dict[str, List[str]]]:
def collect_sources(source_bundle: SourceBundle) -> Dict[str, Dict[str, List[str]]]:
"""
Returns a compiler-ready mapping of solidity source files in source_dir (recursive)
Walks source_dir top-down to the bottom filepath of each subdirectory recursively
and filtrates by __source_filter, setting values into `source_paths`.
"""
source_paths: Dict[str, Dict[str, List[str]]] = dict()
source_walker: Iterator = os.walk(top=str(source_dir), topdown=True)
# Collect single directory
for root, dirs, files in source_walker:
# Collect files in source dir
for filename in filter(source_filter, files):
path = Path(root) / filename
source_paths[filename] = dict(urls=[str(path.resolve(strict=True))])
SOLC_LOGGER.debug(f"Collecting solidity source {path}")
SOLC_LOGGER.info(f"Collected {len(source_paths)} solidity source files at {source_dir}")
for source_dir in source_bundle.source_dirs:
source_walker: Iterator = list(os.walk(top=str(source_dir.resolve(strict=True)), topdown=True)) # TODO: Remove list caster
# Collect single directory
for root, dirs, files in source_walker:
# Collect files in source dir
mapped_path = str(root).replace(str(source_bundle), '')
for filename in filter(source_filter, files):
path = Path(root) / filename
source_paths[filename] = dict(urls=[mapped_path])
SOLC_LOGGER.debug(f"Collecting solidity source {path}")
SOLC_LOGGER.info(f"Collected {len(source_paths)} solidity source files at {source_bundle}")
return source_paths

24
nucypher/blockchain/eth/sol/compile/compile.py
View File

@ -17,37 +17,39 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
import itertools
from pathlib import Path
from typing import Tuple, List, Dict
from cytoolz.dicttoolz import merge, merge_with
from pathlib import Path
from nucypher.blockchain.eth.sol.__conf__ import SOLIDITY_COMPILER_VERSION
from nucypher.blockchain.eth.sol.compile.aggregation import merge_contract_outputs
from nucypher.blockchain.eth.sol.compile.collect import collect_sources
from nucypher.blockchain.eth.sol.compile.config import BASE_COMPILER_CONFIGURATION
from nucypher.blockchain.eth.sol.compile.config import ALLOWED_PATHS
from nucypher.blockchain.eth.sol.compile.solc import __execute
from nucypher.blockchain.eth.sol.compile.types import VersionString, VersionedContractOutputs, CompiledContractOutputs
from nucypher.blockchain.eth.sol.compile.types import (
VersionString,
VersionedContractOutputs,
CompiledContractOutputs,
SourceBundle
)
def compile_sources(solidity_source_dir: Path, version_check: bool = True) -> Dict:
def compile_sources(source_bundle: SourceBundle, version_check: bool = True) -> Dict:
"""Compiled solidity contracts for a single source directory"""
sources = collect_sources(source_dir=solidity_source_dir)
sources = collect_sources(source_bundle=source_bundle)
solc_configuration = merge(BASE_COMPILER_CONFIGURATION, dict(sources=sources)) # do not mutate
allowed_paths = ','.join(list(set(str(p) for p in ALLOWED_PATHS))) # unique
ignore_version_check: bool = not version_check
version: VersionString = VersionString(SOLIDITY_COMPILER_VERSION) if ignore_version_check else None
compiler_output = __execute(compiler_version=version, input_config=solc_configuration, allowed_paths=allowed_paths)
compiler_output = __execute(compiler_version=version, input_config=solc_configuration, base_path=str(source_bundle.import_root))
return compiler_output
def multiversion_compile(solidity_source_dirs: Tuple[Path, ...], compiler_version_check: bool = True) -> VersionedContractOutputs:
def multiversion_compile(source_bundles: Tuple[SourceBundle, ...], compiler_version_check: bool = True) -> VersionedContractOutputs:
"""Compile contracts from `source_dirs` and aggregate the resulting source contract outputs by version"""
raw_compiler_results: List[CompiledContractOutputs] = list()
for source_dir in solidity_source_dirs:
compile_result = compile_sources(solidity_source_dir=source_dir, version_check=compiler_version_check)
for bundle in source_bundles:
compile_result = compile_sources(source_bundle=bundle, version_check=compiler_version_check)
raw_compiler_results.append(compile_result['contracts'])
raw_compiled_contracts = itertools.chain.from_iterable(output.values() for output in raw_compiler_results)
versioned_contract_outputs = merge_with(merge_contract_outputs, *raw_compiled_contracts)

35
nucypher/blockchain/eth/sol/compile/config.py
View File

@ -18,8 +18,6 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
from typing import List, Dict
from nucypher.blockchain.eth.sol.compile.constants import NUCYPHER_CONTRACTS_DIR, ZEPPELIN_DIR, ARAGON_DIR, ARAGON, ZEPPELIN, \
SOLIDITY_SOURCE_ROOT, TEST_SOLIDITY_SOURCE_ROOT
from nucypher.blockchain.eth.sol.compile.types import CompilerConfiguration
@ -33,23 +31,15 @@ WARNING: Do not change these values unless you know what you are doing.
"""
# Debug
# -----
# How to treat revert (and require) reason strings.
# "default", "strip", "debug" and "verboseDebug".
# "default" does not inject compiler-generated revert strings and keeps user-supplied ones.
# "strip" removes all revert strings (if possible, i.e. if literals are used) keeping side-effects
# "debug" injects strings for compiler-generated internal reverts, implemented for ABI encoders V1 and V2 for now.
# "verboseDebug" even appends further information to user-supplied revert strings (not yet implemented)
DEBUG = 'default'
# Hardcoded for added sanity.
# New top-level contract source directories must be listed here.
# Paths can be commented out to prevent default permission.
# In tests, this list can be mutated to temporarily allow compilation
# of source files that are typically not permitted.
ALLOWED_PATHS = [
SOLIDITY_SOURCE_ROOT,
TEST_SOLIDITY_SOURCE_ROOT
]
# DEBUG = 'default'
# Source code language. Currently supported are "Solidity" and "Yul".
LANGUAGE: str = 'Solidity'
@ -59,7 +49,7 @@ EVM_VERSION: str = 'berlin'
# File level compiler outputs (needs empty string as contract name):
FILE_OUTPUTS: List[str] = [
# 'ast' # AST of all source files # TODO: Handle AST for static analysis
'ast' # AST of all source files
# 'legacyAST' # legacy AST of all source files
]
@ -88,10 +78,8 @@ CONTRACT_OUTPUTS: List[str] = [
# 'ewasm.wasm', # eWASM binary format (not supported at the moment)
]
# Optional
# Switch optimizer components on or off in detail.
# The "enabled" switch above provides two defaults which can be tweaked here.
# Optimizer Details - Switch optimizer components on or off in detail.
# The "enabled" switch above provides two defaults which can be tweaked here (yul, and ...).
OPTIMIZER_DETAILS = dict(
peephole=True, # The peephole optimizer is always on if no details are given (switch it off here).
jumpdestRemover=True, # The unused jumpdest remover is always on if no details are given (switch it off here).
@ -103,7 +91,7 @@ OPTIMIZER_DETAILS = dict(
# The new Yul optimizer. Mostly operates on the code of ABIEncoderV2 and inline assembly.
# It is activated together with the global optimizer setting and can be deactivated here.
# Before Solidity 0.6.0 it had to be activated through this switch. Also see 'yulDetails options'.
yul=False
yul=True
)
# Optimize for how many times you intend to run the code.
@ -117,19 +105,14 @@ OPTIMIZER_SETTINGS = dict(
# details=OPTIMIZER_DETAILS # Optional - If "details" is given, "enabled" can be omitted.
)
IMPORT_REMAPPING: List[str] = [
f"contracts={NUCYPHER_CONTRACTS_DIR.resolve()}",
f"{ZEPPELIN}={ZEPPELIN_DIR.resolve()}",
f"{ARAGON}={ARAGON_DIR.resolve()}",
]
# Complete compiler settings
COMPILER_SETTINGS: Dict = dict(
remappings=IMPORT_REMAPPING,
optimizer=OPTIMIZER_SETTINGS,
evmVersion=EVM_VERSION,
outputSelection={"*": {"*": CONTRACT_OUTPUTS, "": FILE_OUTPUTS}} # all contacts(*), all files("")
)
# Base configuration for programmatic usage
BASE_COMPILER_CONFIGURATION = CompilerConfiguration(
language=LANGUAGE,
settings=COMPILER_SETTINGS,

30
nucypher/blockchain/eth/sol/compile/constants.py
View File

@ -19,29 +19,24 @@ along with nucypher. If not, see <https://www.gnu.org/licenses/>.
import re
from logging import Logger
from pathlib import Path
from typing import Tuple, Pattern
from typing import Tuple, Pattern, NamedTuple, List
from nucypher.config.constants import NUCYPHER_TEST_DIR
# Logging
SOLC_LOGGER = Logger("solidity-compilation")
# Vocabulary
CONTRACTS = 'contracts'
ZEPPELIN: str = 'zeppelin'
ARAGON: str = 'aragon'
TEST_SOLIDITY_SOURCE_ROOT: Path = Path(NUCYPHER_TEST_DIR).parent / CONTRACTS / CONTRACTS
from nucypher.blockchain.eth import sol
SOLIDITY_SOURCE_ROOT: Path = Path(sol.__file__).parent / 'source'
# Import Remapping
ZEPPELIN_DIR: Path = SOLIDITY_SOURCE_ROOT / ZEPPELIN
ARAGON_DIR: Path = SOLIDITY_SOURCE_ROOT / ARAGON
NUCYPHER_CONTRACTS_DIR: Path = SOLIDITY_SOURCE_ROOT / 'contracts'
# Do not compile contracts containing...
IGNORE_CONTRACT_PREFIXES: Tuple[str, ...] = (
'Abstract',
@ -50,19 +45,4 @@ IGNORE_CONTRACT_PREFIXES: Tuple[str, ...] = (
DEFAULT_VERSION_STRING: str = 'v0.0.0' # for both compiler and devdoc versions (must fully match regex pattern below)
# RE pattern for matching solidity source compile version specification in devdoc details.
DEVDOC_VERSION_PATTERN: Pattern = re.compile(r"""
\A # Anchor must be first
\| # Anchored pipe literal at beginning of version definition
( # Start Inner capture group
v # Capture version starting from symbol v
\d+ # At least one digit of major version
\. # Digits splitter
\d+ # At least one digit of minor version
\. # Digits splitter
\d+ # At least one digit of patch
) # End of capturing
\| # Anchored end of version definition |
\Z # Anchor must be the end of the match
""", re.VERBOSE)
# TODO: Move closer to usage

8
nucypher/blockchain/eth/sol/compile/solc.py
View File

@ -20,14 +20,14 @@ from logging import Logger
from typing import Dict
from nucypher.blockchain.eth.sol.compile.config import IMPORT_REMAPPING, OPTIMIZER_RUNS, ALLOWED_PATHS
from nucypher.blockchain.eth.sol.compile.config import OPTIMIZER_RUNS
from nucypher.blockchain.eth.sol.compile.constants import SOLC_LOGGER
from nucypher.blockchain.eth.sol.compile.exceptions import CompilationError
from nucypher.blockchain.eth.sol.compile.types import VersionString
from nucypher.exceptions import DevelopmentInstallationRequired
def __execute(compiler_version: VersionString, input_config: Dict, allowed_paths: str):
def __execute(compiler_version: VersionString, input_config: Dict, base_path: str):
"""Executes the solcx command and underlying solc wrapper"""
log = Logger('execute-solcx')
@ -40,11 +40,11 @@ def __execute(compiler_version: VersionString, input_config: Dict, allowed_paths
# Prepare Solc Command
solc_binary_path: str = get_executable(version=compiler_version)
SOLC_LOGGER.info(f"Compiling with import remappings {' '.join(IMPORT_REMAPPING)} and allowed paths {ALLOWED_PATHS}")
SOLC_LOGGER.info(f"Compiling with base path") # TODO: Add base path
# Execute Compilation
try:
compiler_output = compile_standard(input_data=input_config, allow_paths=allowed_paths)
compiler_output = compile_standard(input_data=input_config, base_path=base_path)
except FileNotFoundError:
raise CompilationError("The solidity compiler is not at the specified path. "
"Check that the file exists and is executable.")

10
nucypher/blockchain/eth/sol/compile/types.py
View File

@ -14,9 +14,8 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with nucypher. If not, see <https://www.gnu.org/licenses/>.
"""
from typing import Dict, List, Union, NewType
from pathlib import Path
from typing import Dict, List, Union, NewType, NamedTuple, Tuple, Optional
class ABI(Dict):
@ -42,3 +41,8 @@ class CompilerConfiguration(Dict):
language: str
sources: Dict[str, Dict[str, str]]
settings: Dict
class SourceBundle(NamedTuple):
source_dirs: Tuple[Path]
import_root: Optional[Path] = None

14
tests/acceptance/blockchain/interfaces/test_handle_multiversion_contracts.py
View File

@ -17,8 +17,7 @@
from pathlib import Path
from nucypher.blockchain.eth.sol.compile.config import ALLOWED_PATHS
from nucypher.blockchain.eth.sol.compile.types import SourceBundle
from nucypher.blockchain.eth.interfaces import BlockchainDeployerInterface, BlockchainInterfaceFactory
from nucypher.blockchain.eth.registry import InMemoryContractRegistry
from nucypher.crypto.powers import TransactingPower
@ -29,13 +28,14 @@ def test_deployer_interface_multiversion_contract():
# Prepare compiler
base_dir = Path(__file__).parent / 'contracts' / 'multiversion'
v1_dir, v2_dir = base_dir / 'v1', base_dir / 'v2'
v1_dir, v2_dir = Path(base_dir / 'v1'), Path(base_dir / 'v2')
source_dirs = v1_dir, v2_dir
# I am a contract administrator and I an compiling a new updated version of an existing contract...
# Represents "Manually hardcoding" a new permitted compile path in compile.py
# and new source directory on BlockchainDeployerInterface.SOURCES.
ALLOWED_PATHS.append(base_dir)
BlockchainDeployerInterface.SOURCES = (v1_dir, v2_dir)
# Represents "Manually hardcoding" a new source directory on BlockchainDeployerInterface.SOURCES.
BlockchainDeployerInterface.SOURCES = (
SourceBundle(source_dirs=source_dirs, import_root=base_dir),
)
# Prepare chain
BlockchainInterfaceFactory._interfaces.clear()

6
tests/contracts/test_contracts_upgradeability.py
View File

@ -93,13 +93,13 @@ def deploy_earliest_contract(blockchain_interface: BlockchainDeployerInterface,
pass # Skip errors related to initialization
@pytest.mark.skip('GH 403') # FIXME
# FIXME: Needs Completion
def test_upgradeability(temp_dir_path, token_economics):
# Prepare remote source for compilation
download_github_dir(GITHUB_SOURCE_LINK, temp_dir_path)
# Prepare the blockchain
provider_uri = 'tester://pyevm/2'
provider_uri = 'tester://pyevm/2' # TODO: Testerchain caching Issues
try:
blockchain_interface = BlockchainDeployerInterface(provider_uri=provider_uri,
gas_strategy=free_gas_price_strategy)
@ -143,6 +143,6 @@ def test_upgradeability(temp_dir_path, token_economics):
adjudicator_deployer.upgrade(contract_version="latest", confirmations=0)
finally:
# Unregister interface
# Unregister interface # TODO: Move to method?
with contextlib.suppress(KeyError):
del BlockchainInterfaceFactory._interfaces[provider_uri]