From 97d35bae7e610aa618db11f6185b052867c7ff3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=BA=C3=B1ez?= Date: Wed, 6 Nov 2019 12:50:53 +0100 Subject: [PATCH 1/7] Accept also CSV format for the input allocation file --- nucypher/blockchain/eth/actors.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/nucypher/blockchain/eth/actors.py b/nucypher/blockchain/eth/actors.py index 9e230d99f..1c7a88811 100644 --- a/nucypher/blockchain/eth/actors.py +++ b/nucypher/blockchain/eth/actors.py @@ -16,6 +16,7 @@ along with nucypher. If not, see . """ +import csv import json import os from datetime import datetime @@ -383,13 +384,20 @@ class ContractAdministrator(NucypherTokenActor): emitter: StdoutEmitter = None, ) -> Dict[str, dict]: """ - The allocation file is a JSON file containing a list of allocations. Each allocation has a: + The allocation file is a CSV file containing a list of allocations. Each allocation has a: * 'beneficiary_address': Checksum address of the beneficiary * 'name': User-friendly name of the beneficiary (Optional) * 'amount': Amount of tokens locked, in NuNits * 'duration_seconds': Lock duration expressed in seconds - Example allocation file: + Example allocation file in CSV format: + + "beneficiary_address","name","amount","duration_seconds" + "0xdeadbeef","H. E. Pennypacker",100,31536000 + "0xabced120","",133432,31536000 + "0xf7aefec2","",999,31536000 + + Example allocation file in JSON format: [ {'beneficiary_address': '0xdeadbeef', 'name': 'H. E. Pennypacker', 'amount': 100, 'duration_seconds': 31536000}, {'beneficiary_address': '0xabced120', 'amount': 133432, 'duration_seconds': 31536000}, @@ -518,11 +526,18 @@ class ContractAdministrator(NucypherTokenActor): @staticmethod def __read_allocation_data(filepath: str) -> list: with open(filepath, 'r') as allocation_file: - data = allocation_file.read() - try: - allocation_data = json.loads(data) - except JSONDecodeError: - raise + if filepath.endswith(".csv"): + allocation_data = list() + reader = csv.DictReader(allocation_file) + for entry in reader: + entry['amount'], entry['duration_seconds'] = int(entry['amount']), int(entry['duration_seconds']) + allocation_data.append(entry) + else: + data = allocation_file.read() + try: + allocation_data = json.loads(data) + except JSONDecodeError: + raise return allocation_data def deploy_beneficiaries_from_file(self, From e341aace88d9fda9f1ad6d5fa3f58ed70dac0b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=BA=C3=B1ez?= Date: Wed, 6 Nov 2019 16:47:14 +0100 Subject: [PATCH 2/7] Check if beneficiary is already enrolled in allocation registry prior deployment --- nucypher/blockchain/eth/actors.py | 6 ++++++ nucypher/blockchain/eth/registry.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/nucypher/blockchain/eth/actors.py b/nucypher/blockchain/eth/actors.py index 1c7a88811..6174db0d2 100644 --- a/nucypher/blockchain/eth/actors.py +++ b/nucypher/blockchain/eth/actors.py @@ -442,6 +442,12 @@ class ContractAdministrator(NucypherTokenActor): if emitter: emitter.echo(f"Saved allocation template file to {template_filepath}", color='blue', bold=True) + already_enrolled = [a['beneficiary_address'] for a in allocations + if allocation_registry.is_beneficiary_enrolled(a['beneficiary_address'])] + if already_enrolled: + raise ValueError(f"The following beneficiaries are already enrolled in allocation registry " + f"({allocation_registry.filepath}): {already_enrolled}") + # Deploy each allocation contract with click.progressbar(length=total_deployment_transactions, label="Allocation progress", diff --git a/nucypher/blockchain/eth/registry.py b/nucypher/blockchain/eth/registry.py index 22c8acb3a..3658a9b07 100644 --- a/nucypher/blockchain/eth/registry.py +++ b/nucypher/blockchain/eth/registry.py @@ -404,7 +404,7 @@ class AllocationRegistry(LocalContractRegistry): try: _ = self.search(beneficiary_address=beneficiary_address) return True - except self.UnknownBeneficiary: + except (self.UnknownBeneficiary, self.NoAllocationRegistry): return False # TODO: Tests! From ce9e1d15116ee086944d09c50e9d3dc0b54455f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=BA=C3=B1ez?= Date: Thu, 7 Nov 2019 14:06:49 +0100 Subject: [PATCH 3/7] Try our best to adapt allocation input data --- nucypher/blockchain/eth/actors.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nucypher/blockchain/eth/actors.py b/nucypher/blockchain/eth/actors.py index 6174db0d2..5e181312d 100644 --- a/nucypher/blockchain/eth/actors.py +++ b/nucypher/blockchain/eth/actors.py @@ -544,6 +544,10 @@ class ContractAdministrator(NucypherTokenActor): allocation_data = json.loads(data) except JSONDecodeError: raise + + for entry in allocation_data: + entry['beneficiary_address'] = to_checksum_address(entry['beneficiary_address']) + return allocation_data def deploy_beneficiaries_from_file(self, From 9613aeb6a219be38d26c651f6a788a5ed15beb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=BA=C3=B1ez?= Date: Thu, 7 Nov 2019 14:07:52 +0100 Subject: [PATCH 4/7] Improve collection of allocation receipts --- nucypher/blockchain/eth/actors.py | 21 ++++++------ nucypher/blockchain/eth/deployers.py | 33 ++++++++++--------- .../deployers/test_deploy_preallocations.py | 6 ++-- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/nucypher/blockchain/eth/actors.py b/nucypher/blockchain/eth/actors.py index 5e181312d..f57a03ec8 100644 --- a/nucypher/blockchain/eth/actors.py +++ b/nucypher/blockchain/eth/actors.py @@ -32,7 +32,7 @@ from constant_sorrow.constants import ( NO_WORKER_ASSIGNED, ) from eth_tester.exceptions import TransactionFailed -from eth_utils import keccak, is_checksum_address +from eth_utils import keccak, is_checksum_address, to_checksum_address from twisted.logger import Logger from nucypher.blockchain.economics import TokenEconomics, StandardTokenEconomics, TokenEconomicsFactory @@ -469,16 +469,17 @@ class ContractAdministrator(NucypherTokenActor): bar._last_line = None bar.render_progress() - deployer = self.deploy_preallocation_escrow(allocation_registry=allocation_registry, - progress=bar) - amount = allocation['amount'] duration = allocation['duration_seconds'] + try: - receipts = deployer.deliver(value=amount, - duration=duration, - beneficiary_address=beneficiary, - progress=bar) + deployer = self.deploy_preallocation_escrow(allocation_registry=allocation_registry, + progress=bar) + + deployer.deliver(value=amount, + duration=duration, + beneficiary_address=beneficiary, + progress=bar) except TransactionFailed as e: if crash_on_failure: raise @@ -487,7 +488,7 @@ class ContractAdministrator(NucypherTokenActor): continue else: - allocation_receipts[beneficiary] = receipts + allocation_receipts[beneficiary] = deployer.deployment_receipts allocation_contract_address = deployer.contract_address self.log.info(f"Created {deployer.contract_name} contract at {allocation_contract_address} " f"for beneficiary {beneficiary}.") @@ -506,7 +507,7 @@ class ContractAdministrator(NucypherTokenActor): if emitter: blockchain = BlockchainInterfaceFactory.get_interface() paint_contract_deployment(contract_name=deployer.contract_name, - receipts=receipts, + receipts=deployer.deployment_receipts, contract_address=deployer.contract_address, emitter=emitter, chain_name=blockchain.client.chain_name, diff --git a/nucypher/blockchain/eth/deployers.py b/nucypher/blockchain/eth/deployers.py index de2e8366d..9e01e09af 100644 --- a/nucypher/blockchain/eth/deployers.py +++ b/nucypher/blockchain/eth/deployers.py @@ -16,6 +16,7 @@ along with nucypher. If not, see . """ +from collections import OrderedDict from typing import Tuple, Dict from constant_sorrow.constants import CONTRACT_NOT_DEPLOYED, NO_DEPLOYER_CONFIGURED, NO_BENEFICIARY @@ -73,7 +74,7 @@ class BaseContractDeployer: # Defaults # self.registry = registry - self.deployment_receipts = dict() + self.deployment_receipts = OrderedDict() self._contract = CONTRACT_NOT_DEPLOYED self.__proxy_contract = NotImplemented self.__deployer_address = deployer_address @@ -794,7 +795,7 @@ class PreallocationEscrowDeployer(BaseContractDeployer, UpgradeableContractMixin agency = PreallocationEscrowAgent contract_name = agency.registry_contract_name - deployment_steps = ('contract_deployment', ) + deployment_steps = ('contract_deployment', 'approve_transfer', 'initial_deposit', 'transfer_ownership') _router_deployer = StakingInterfaceRouterDeployer __allocation_registry = AllocationRegistry @@ -828,16 +829,18 @@ class PreallocationEscrowDeployer(BaseContractDeployer, UpgradeableContractMixin sender_address=self.deployer_address, payload=payload) self.__beneficiary_address = checksum_address + self.deployment_receipts.update({self.deployment_steps[3]: transfer_owner_receipt}) return transfer_owner_receipt - def initial_deposit(self, value: int, duration_seconds: int, progress=None) -> dict: + def initial_deposit(self, value: int, duration_seconds: int, progress=None): """Allocate an amount of tokens with lock time in seconds, and transfer ownership to the beneficiary""" # Approve - allocation_receipts = dict() - approve_function = self.token_contract.functions.approve(self.contract.address, value) + approve_function = self.token_contract.functions.approve(self.contract.address, value) approve_receipt = self.blockchain.send_transaction(contract_function=approve_function, sender_address=self.deployer_address) # TODO: Gas - allocation_receipts['approve'] = approve_receipt + + self.deployment_receipts.update({self.deployment_steps[1]: approve_receipt}) + if progress: progress.update(1) # Deposit @@ -848,13 +851,10 @@ class PreallocationEscrowDeployer(BaseContractDeployer, UpgradeableContractMixin sender_address=self.deployer_address, payload=args) - # TODO: Do something with allocation_receipts. Perhaps it should be returned instead of only the last receipt. - allocation_receipts['initial_deposit'] = deposit_receipt + self.deployment_receipts.update({self.deployment_steps[2]: deposit_receipt}) if progress: progress.update(1) - return deposit_receipt - def enroll_principal_contract(self): if self.__beneficiary_address is NO_BENEFICIARY: raise self.ContractDeploymentError("No beneficiary assigned to {}".format(self.contract.address)) @@ -863,7 +863,7 @@ class PreallocationEscrowDeployer(BaseContractDeployer, UpgradeableContractMixin contract_abi=self.contract.abi) @validate_checksum_address - def deliver(self, value: int, duration: int, beneficiary_address: str, progress=None) -> dict: + def deliver(self, value: int, duration: int, beneficiary_address: str, progress=None): """ Transfer allocated tokens and hand-off the contract to the beneficiary. @@ -874,12 +874,11 @@ class PreallocationEscrowDeployer(BaseContractDeployer, UpgradeableContractMixin """ - deposit_receipt = self.initial_deposit(value=value, duration_seconds=duration, progress=progress) - assign_receipt = self.assign_beneficiary(checksum_address=beneficiary_address) + self.initial_deposit(value=value, duration_seconds=duration, progress=progress) + self.assign_beneficiary(checksum_address=beneficiary_address) if progress: progress.update(1) self.enroll_principal_contract() - return dict(deposit_receipt=deposit_receipt, assign_receipt=assign_receipt) def deploy(self, initial_deployment: bool = True, gas_limit: int = None, progress=None) -> dict: """Deploy a new instance of PreallocationEscrow to the blockchain.""" @@ -892,12 +891,14 @@ class PreallocationEscrowDeployer(BaseContractDeployer, UpgradeableContractMixin router_contract.address, self.token_contract.address) - preallocation_escrow_contract, deploy_receipt = self.blockchain.deploy_contract(*args, gas_limit=gas_limit, enroll=False) + preallocation_escrow_contract, deploy_receipt = self.blockchain.deploy_contract(*args, + gas_limit=gas_limit, + enroll=False) if progress: progress.update(1) self._contract = preallocation_escrow_contract - self.deployment_receipts.update({'deployment': deploy_receipt}) + self.deployment_receipts.update({self.deployment_steps[0]: deploy_receipt}) return deploy_receipt def get_contract_abi(self): diff --git a/tests/blockchain/eth/entities/deployers/test_deploy_preallocations.py b/tests/blockchain/eth/entities/deployers/test_deploy_preallocations.py index 78d0e8df1..d1bf47e5c 100644 --- a/tests/blockchain/eth/entities/deployers/test_deploy_preallocations.py +++ b/tests/blockchain/eth/entities/deployers/test_deploy_preallocations.py @@ -55,14 +55,12 @@ def test_deploy_and_allocate(agency, token_economics, test_registry): assert token_agent.get_balance(address=origin) > 1 # Start allocating tokens - deposit_receipts, approve_hashes = list(), dict() for address, deployer in deployments.items(): assert deployer.deployer_address == origin - deposit_receipt = deployer.initial_deposit(value=allocation, duration_seconds=token_economics.maximum_rewarded_periods) - deposit_receipts.append(deposit_receipt) + deployer.initial_deposit(value=allocation, duration_seconds=token_economics.maximum_rewarded_periods) beneficiary = random.choice(testerchain.unassigned_accounts) _assign_receipt = deployer.assign_beneficiary(beneficiary) - assert len(deposit_receipts) == number_of_deployments == len(deployments) + assert number_of_deployments == len(deployments) From 0935e9355ea48b7337eeac1a29806de262856dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=BA=C3=B1ez?= Date: Thu, 7 Nov 2019 23:56:18 +0100 Subject: [PATCH 5/7] Fix bug in method that saves deployment receipts as a file --- nucypher/blockchain/eth/actors.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nucypher/blockchain/eth/actors.py b/nucypher/blockchain/eth/actors.py index f57a03ec8..3f7811321 100644 --- a/nucypher/blockchain/eth/actors.py +++ b/nucypher/blockchain/eth/actors.py @@ -518,7 +518,7 @@ class ContractAdministrator(NucypherTokenActor): if emitter: paint_deployed_allocations(emitter, allocated, failed) - csv_filename = f'allocations-{self.deployer_address[:6]}-{maya.now().epoch}.csv' + csv_filename = f'allocation-summary-{self.deployer_address[:6]}-{maya.now().epoch}.csv' csv_filepath = os.path.join(parent_path, csv_filename) write_deployed_allocations_to_csv(csv_filepath, allocated, failed) if emitter: @@ -575,12 +575,12 @@ class ContractAdministrator(NucypherTokenActor): os.makedirs(DEFAULT_CONFIG_ROOT, exist_ok=True) with open(filepath, 'w') as file: data = dict() - for contract_name, receipts in receipts.items(): + for contract_name, contract_receipts in receipts.items(): contract_records = dict() - for tx_name, receipt in receipts.items(): + for tx_name, receipt in contract_receipts.items(): # Formatting - receipt = {item: str(result) for item, result in receipt.items()} - contract_records.update({tx_name: receipt for tx_name in receipts}) + pretty_receipt = {item: str(result) for item, result in receipt.items()} + contract_records[tx_name] = pretty_receipt data[contract_name] = contract_records data = json.dumps(data, indent=4) file.write(data) From 16c08de17b12c151bca57d134768195e5bbf181a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=BA=C3=B1ez?= Date: Mon, 11 Nov 2019 13:11:18 +0100 Subject: [PATCH 6/7] Cleanup method for reading allocation data --- nucypher/blockchain/eth/actors.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/nucypher/blockchain/eth/actors.py b/nucypher/blockchain/eth/actors.py index 3f7811321..0c11173a4 100644 --- a/nucypher/blockchain/eth/actors.py +++ b/nucypher/blockchain/eth/actors.py @@ -384,13 +384,13 @@ class ContractAdministrator(NucypherTokenActor): emitter: StdoutEmitter = None, ) -> Dict[str, dict]: """ - The allocation file is a CSV file containing a list of allocations. Each allocation has a: + The allocation file contains a list of allocations, each of them composed of: * 'beneficiary_address': Checksum address of the beneficiary * 'name': User-friendly name of the beneficiary (Optional) * 'amount': Amount of tokens locked, in NuNits * 'duration_seconds': Lock duration expressed in seconds - Example allocation file in CSV format: + It accepts both CSV and JSON formats. Example allocation file in CSV format: "beneficiary_address","name","amount","duration_seconds" "0xdeadbeef","H. E. Pennypacker",100,31536000 @@ -534,20 +534,17 @@ class ContractAdministrator(NucypherTokenActor): def __read_allocation_data(filepath: str) -> list: with open(filepath, 'r') as allocation_file: if filepath.endswith(".csv"): - allocation_data = list() reader = csv.DictReader(allocation_file) - for entry in reader: - entry['amount'], entry['duration_seconds'] = int(entry['amount']), int(entry['duration_seconds']) - allocation_data.append(entry) - else: + allocation_data = list(reader) + else: # Assume it's JSON by default data = allocation_file.read() - try: - allocation_data = json.loads(data) - except JSONDecodeError: - raise + allocation_data = json.loads(data) + # Pre-process allocation data for entry in allocation_data: entry['beneficiary_address'] = to_checksum_address(entry['beneficiary_address']) + entry['amount'] = int(entry['amount']) + entry['duration_seconds'] = int(entry['duration_seconds']) return allocation_data From f1cc8db503f6b6655d2d16a0fc4bd2e9e4c59622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=BA=C3=B1ez?= Date: Tue, 12 Nov 2019 00:25:47 +0100 Subject: [PATCH 7/7] Responded to RFCs in PR #1446 --- nucypher/blockchain/eth/actors.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nucypher/blockchain/eth/actors.py b/nucypher/blockchain/eth/actors.py index 0c11173a4..fd00acb5a 100644 --- a/nucypher/blockchain/eth/actors.py +++ b/nucypher/blockchain/eth/actors.py @@ -483,7 +483,10 @@ class ContractAdministrator(NucypherTokenActor): except TransactionFailed as e: if crash_on_failure: raise - self.log.debug(f"Failed allocation transaction for {NU.from_nunits(amount)} to {beneficiary}: {e}") + message = f"Failed allocation transaction for {NU.from_nunits(amount)} to {beneficiary}: {e}" + self.log.debug(message) + if emitter: + emitter.echo(message=message, color='red', bold=True) failed.append(allocation) continue @@ -525,7 +528,6 @@ class ContractAdministrator(NucypherTokenActor): emitter.echo(f"Saved allocation summary CSV to {csv_filepath}", color='blue', bold=True) if failed: - # TODO: More with these failures: send to isolated logfile, and reattempt self.log.critical(f"FAILED TOKEN ALLOCATION - {len(failed)} allocations failed.") return allocation_receipts @@ -537,8 +539,7 @@ class ContractAdministrator(NucypherTokenActor): reader = csv.DictReader(allocation_file) allocation_data = list(reader) else: # Assume it's JSON by default - data = allocation_file.read() - allocation_data = json.loads(data) + allocation_data = json.load(allocation_file) # Pre-process allocation data for entry in allocation_data: @@ -558,7 +559,8 @@ class ContractAdministrator(NucypherTokenActor): receipts = self.deploy_beneficiary_contracts(allocations=allocations, allocation_outfile=allocation_outfile, emitter=emitter, - interactive=interactive) + interactive=interactive, + crash_on_failure=False) # Save transaction metadata receipts_filepath = self.save_deployment_receipts(receipts=receipts, filename_prefix='allocation') if emitter: