Add redundancy POA error middleware that adds/replaces geth poa middleware if ExtraDataLengthError is encountered at any time.

nucypher/blockchain/eth/clients.py

@@ -16,6 +16,7 @@ from nucypher.blockchain.eth.constants import (
     POA_CHAINS,
     PUBLIC_CHAINS,
 )
+from nucypher.blockchain.middleware.poa import POAErrorRedundancyMiddleware
 from nucypher.blockchain.middleware.retry import (
     AlchemyRetryRequestMiddleware,
     InfuraRetryRequestMiddleware,
@@ -97,13 +98,22 @@ class EthereumClient:
         chain_id = self.chain_id
         is_poa = chain_id in POA_CHAINS
 
-        self.log.debug(
+        self.log.info(
             f"Blockchain: {self.chain_name} (chain_id={chain_id}, poa={is_poa})"
         )
         if is_poa:
             # proof-of-authority blockchain
             self.log.info("Injecting POA middleware at layer 0")
-            self.inject_middleware(geth_poa_middleware, layer=0, name="poa")
+            self.inject_middleware(
+                # use naming from redundancy middleware
+                geth_poa_middleware,
+                layer=0,
+                name=POAErrorRedundancyMiddleware.POA_MIDDLEWARE_NAME,
+            )
+
+        # POA error redundancy middleware, just in case
+        self.log.debug("Adding POA redundancy middleware")
+        self.add_middleware(POAErrorRedundancyMiddleware)
 
         # simple cache middleware
         self.log.debug("Adding simple_cache_middleware")

nucypher/blockchain/middleware/poa.py

@@ -0,0 +1,49 @@
+from typing import Any, Callable
+
+from web3 import Web3
+from web3.exceptions import ExtraDataLengthError
+from web3.middleware import geth_poa_middleware
+from web3.types import RPCEndpoint, RPCResponse
+
+from nucypher.utilities.logging import Logger
+
+
+class POAErrorRedundancyMiddleware:
+    """
+    Redundant middleware for replacing already added named poa middleware if ExtraDataLengthError
+    still encountered. Extra layer of defense in case random POA error is observed
+    """
+
+    POA_MIDDLEWARE_NAME = "poa"
+
+    def __init__(
+        self,
+        make_request: Callable[[RPCEndpoint, Any], RPCResponse],
+        w3: Web3,
+        existing_poa_middleware_name: str = POA_MIDDLEWARE_NAME,
+    ):
+        self.w3 = w3
+        self.make_request = make_request
+        self.existing_poa_middleware_name = existing_poa_middleware_name
+        self.logger = Logger(self.__class__.__name__)
+
+    def __call__(self, method, params) -> RPCResponse:
+        try:
+            response = self.make_request(method, params)
+        except ExtraDataLengthError:
+            self.logger.warn(
+                "RPC request failed due to extraData error; re-injecting poa middleware and retrying"
+            )
+            # add / replace existing poa middleware; replacement seems unlikely but just in case
+            if self.w3.middleware_onion.get(self.existing_poa_middleware_name):
+                # we can't have > 1 geth_poa_middleware added (event with different names) so the
+                # removal-then-add is just for safety.
+                self.w3.middleware_onion.remove(self.existing_poa_middleware_name)
+            self.w3.middleware_onion.inject(
+                geth_poa_middleware, layer=0, name=self.existing_poa_middleware_name
+            )
+
+            # try again
+            response = self.make_request(method, params)
+
+        return response