feat: support web3.py v7 (in addition to v6) (#2394)

Co-authored-by: Evan <163005762+evan-quest@users.noreply.github.com>

Author: antazoey

setup.py

@@ -118,15 +118,14 @@
         "urllib3>=2.0.0,<3",
         "watchdog>=3.0,<4",
         # ** Dependencies maintained by Ethereum Foundation **
-        # All version pins dependent on web3[tester]
-        "eth-abi",
-        "eth-account",
-        "eth-typing>=3.5.2,<4",
-        "eth-utils",
-        "hexbytes",
-        "py-geth>=5.1.0,<6",
+        "eth-abi>=5.1.0,<6",
+        "eth-account>=0.11.3,<0.14",
+        "eth-typing>=3.5.2,<6",
+        "eth-utils>=2.1.0,<6",
+        "hexbytes>=0.3.1,<2",
+        "py-geth>=3.14.0,<6",
         "trie>=3.0.1,<4",  # Peer: stricter pin needed for uv support.
-        "web3[tester]>=6.17.2,<7",
+        "web3[tester]>=6.20.1,<8",
         # ** Dependencies maintained by ApeWorX **
         "eip712>=0.2.10,<0.3",
         "ethpm-types>=0.6.19,<0.7",

src/ape/utils/_web3_compat.py

@@ -0,0 +1,28 @@
+from eth_account import Account as EthAccount
+
+try:
+    # Web3 v7
+    from web3.middleware import ExtraDataToPOAMiddleware  # type: ignore
+except ImportError:
+    from web3.middleware import geth_poa_middleware as ExtraDataToPOAMiddleware  # type: ignore
+
+try:
+    from web3.providers import WebsocketProviderV2 as WebsocketProvider  # type: ignore
+except ImportError:
+    from web3.providers import WebSocketProvider as WebsocketProvider  # type: ignore
+
+
+def sign_hash(msghash, private_key):
+    try:
+        # Web3 v7
+        return EthAccount.unsafe_sign_hash(msghash, private_key)  # type: ignore
+    except AttributeError:
+        # Web3 v6
+        return EthAccount.signHash(msghash, private_key)  # type: ignore
+
+
+__all__ = [
+    "ExtraDataToPOAMiddleware",
+    "sign_hash",
+    "WebsocketProvider",
+]

src/ape_accounts/accounts.py

@@ -18,6 +18,7 @@
 from ape.exceptions import AccountsError
 from ape.logging import logger
 from ape.types.signatures import MessageSignature, SignableMessage, TransactionSignature
+from ape.utils._web3_compat import sign_hash
 from ape.utils.basemodel import ManagerAccessMixin
 from ape.utils.misc import log_instead_of_fail
 from ape.utils.validators import _validate_account_alias, _validate_account_passphrase
@@ -255,7 +256,7 @@ def sign_raw_msghash(self, msghash: HexBytes) -> Optional[MessageSignature]:
         # Also, we have already warned the user about the safety.
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
-            signed_msg = EthAccount.signHash(msghash, self.__key)
+            signed_msg = sign_hash(msghash, self.__key)
 
         return MessageSignature(
             v=signed_msg.v,

src/ape_ethereum/provider.py

@@ -20,17 +20,21 @@
 from pydantic.dataclasses import dataclass
 from requests import HTTPError
 from web3 import HTTPProvider, IPCProvider, Web3
-from web3 import WebsocketProvider as WebSocketProvider
-from web3._utils.http import construct_user_agent
+from web3 import __version__ as web3_version
 from web3.exceptions import ContractLogicError as Web3ContractLogicError
 from web3.exceptions import (
     ExtraDataLengthError,
     MethodUnavailable,
     TimeExhausted,
     TransactionNotFound,
 )
+
+try:
+    from web3.exceptions import Web3RPCError
+except ImportError:
+    Web3RPCError = ValueError  # type: ignore
+
 from web3.gas_strategies.rpc import rpc_gas_price_strategy
-from web3.middleware import geth_poa_middleware as ExtraDataToPOAMiddleware
 from web3.middleware.validation import MAX_EXTRADATA_LENGTH
 from web3.providers import AutoProvider
 from web3.providers.auto import load_provider_from_environment
@@ -59,6 +63,7 @@
 from ape.types.events import ContractLog, LogFilter
 from ape.types.gas import AutoGasLimit
 from ape.types.trace import SourceTraceback
+from ape.utils._web3_compat import ExtraDataToPOAMiddleware, WebsocketProvider
 from ape.utils.basemodel import ManagerAccessMixin
 from ape.utils.misc import DEFAULT_MAX_RETRIES_TX, gas_estimation_error_message, to_int
 from ape_ethereum._print import CONSOLE_ADDRESS, console_contract
@@ -124,6 +129,24 @@ def assert_web3_provider_uri_env_var_not_set():
     )
 
 
+def _post_send_transaction(tx: TransactionAPI, receipt: ReceiptAPI):
+    """Execute post-transaction ops"""
+
+    # TODO: Optional configuration?
+    if tx.receiver and Address(tx.receiver).is_contract:
+        # Look for and print any contract logging
+        try:
+            receipt.show_debug_logs()
+        except TransactionNotFound:
+            # Receipt never published. Likely failed.
+            pass
+        except Exception as err:
+            # Avoid letting debug logs causes program crashes.
+            logger.debug(f"Unable to show debug logs: {err}")
+
+    logger.info(f"Confirmed {receipt.txn_hash} (total fees paid = {receipt.total_fees_paid})")
+
+
 class Web3Provider(ProviderAPI, ABC):
     """
     A base provider mixin class that uses the
@@ -160,7 +183,7 @@ def post_tx_hook(send_tx):
             @wraps(send_tx)
             def send_tx_wrapper(self, txn: TransactionAPI) -> ReceiptAPI:
                 receipt = send_tx(self, txn)
-                self._post_send_transaction(txn, receipt)
+                _post_send_transaction(txn, receipt)
                 return receipt
 
             return send_tx_wrapper
@@ -1006,35 +1029,9 @@ def prepare_transaction(self, txn: TransactionAPI) -> TransactionAPI:
     def send_transaction(self, txn: TransactionAPI) -> ReceiptAPI:
         vm_err = None
         txn_data = None
-        txn_hash = None
         try:
-            if txn.sender is not None and txn.signature is None:
-                # Missing signature, user likely trying to use an unlocked account.
-                attempt_send = True
-                if (
-                    self.network.is_dev
-                    and txn.sender not in self.account_manager.test_accounts._impersonated_accounts
-                ):
-                    try:
-                        self.account_manager.test_accounts.impersonate_account(txn.sender)
-                    except NotImplementedError:
-                        # Unable to impersonate. Try sending as raw-tx.
-                        attempt_send = False
-
-                if attempt_send:
-                    # For some reason, some nodes have issues with integer-types.
-                    txn_data = {
-                        k: to_hex(v) if isinstance(v, int) else v
-                        for k, v in txn.model_dump(by_alias=True, mode="json").items()
-                    }
-                    tx_params = cast(TxParams, txn_data)
-                    txn_hash = to_hex(self.web3.eth.send_transaction(tx_params))
-                # else: attempt raw tx
-
-            if txn_hash is None:
-                txn_hash = to_hex(self.web3.eth.send_raw_transaction(txn.serialize_transaction()))
-
-        except (ValueError, Web3ContractLogicError) as err:
+            txn_hash = self._send_transaction(txn)
+        except (Web3RPCError, Web3ContractLogicError) as err:
             vm_err = self.get_virtual_machine_error(
                 err, txn=txn, set_ape_traceback=txn.raise_on_revert
             )
@@ -1102,22 +1099,35 @@ def send_transaction(self, txn: TransactionAPI) -> ReceiptAPI:
 
         return receipt
 
-    def _post_send_transaction(self, tx: TransactionAPI, receipt: ReceiptAPI):
-        """Execute post-transaction ops"""
+    def _send_transaction(self, txn: TransactionAPI) -> str:
+        txn_hash = None
+        if txn.sender is not None and txn.signature is None:
+            # Missing signature, user likely trying to use an unlocked account.
+            attempt_send = True
+            if (
+                self.network.is_dev
+                and txn.sender not in self.account_manager.test_accounts._impersonated_accounts
+            ):
+                try:
+                    self.account_manager.test_accounts.impersonate_account(txn.sender)
+                except NotImplementedError:
+                    # Unable to impersonate. Try sending as raw-tx.
+                    attempt_send = False
+
+            if attempt_send:
+                # For some reason, some nodes have issues with integer-types.
+                txn_data = {
+                    k: to_hex(v) if isinstance(v, int) else v
+                    for k, v in txn.model_dump(by_alias=True, mode="json").items()
+                }
+                tx_params = cast(TxParams, txn_data)
+                txn_hash = to_hex(self.web3.eth.send_transaction(tx_params))
+            # else: attempt raw tx
 
-        # TODO: Optional configuration?
-        if tx.receiver and Address(tx.receiver).is_contract:
-            # Look for and print any contract logging
-            try:
-                receipt.show_debug_logs()
-            except TransactionNotFound:
-                # Receipt never published. Likely failed.
-                pass
-            except Exception as err:
-                # Avoid letting debug logs causes program crashes.
-                logger.debug(f"Unable to show debug logs: {err}")
+        if txn_hash is None:
+            txn_hash = to_hex(self.web3.eth.send_raw_transaction(txn.serialize_transaction()))
 
-        logger.info(f"Confirmed {receipt.txn_hash} (total fees paid = {receipt.total_fees_paid})")
+        return txn_hash
 
     def _post_connect(self):
         # Register the console contract for trace enrichment
@@ -1326,9 +1336,7 @@ class EthereumNodeProvider(Web3Provider, ABC):
     name: str = "node"
 
     # NOTE: Appends user-agent to base User-Agent string.
-    request_header: dict = {
-        "User-Agent": construct_user_agent(str(HTTPProvider)),
-    }
+    request_header: dict = {"User-Agent": f"EthereumNodeProvider/web3.py/{web3_version}"}
 
     @property
     def uri(self) -> str:
@@ -1619,7 +1627,7 @@ def _create_web3(
 
         providers.append(lambda: HTTPProvider(endpoint_uri=http, request_kwargs=request_kwargs))
     if ws := ws_uri:
-        providers.append(lambda: WebSocketProvider(endpoint_uri=ws))
+        providers.append(lambda: WebsocketProvider(endpoint_uri=ws))
 
     provider = AutoProvider(potential_providers=providers)
     return Web3(provider)

src/ape_node/provider.py

@@ -13,11 +13,11 @@
 from pydantic import field_validator
 from pydantic_settings import SettingsConfigDict
 from requests.exceptions import ConnectionError
-from web3.middleware import geth_poa_middleware as ExtraDataToPOAMiddleware
 
 from ape.api.config import PluginConfig
 from ape.api.providers import SubprocessProvider, TestProviderAPI
 from ape.logging import LogLevel, logger
+from ape.utils._web3_compat import ExtraDataToPOAMiddleware
 from ape.utils.misc import ZERO_ADDRESS, log_instead_of_fail, raises_not_implemented
 from ape.utils.process import JoinableQueue, spawn
 from ape.utils.testing import (

src/ape_test/accounts.py

@@ -13,6 +13,7 @@
 from ape.api.accounts import TestAccountAPI, TestAccountContainerAPI
 from ape.exceptions import ProviderNotConnectedError, SignatureError
 from ape.types.signatures import MessageSignature, TransactionSignature
+from ape.utils._web3_compat import sign_hash
 from ape.utils.testing import (
     DEFAULT_NUMBER_OF_TEST_ACCOUNTS,
     DEFAULT_TEST_HD_PATH,
@@ -166,7 +167,7 @@ def sign_transaction(
     def sign_raw_msghash(self, msghash: HexBytes) -> MessageSignature:
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
-            signed_msg = EthAccount.signHash(msghash, self.private_key)
+            signed_msg = sign_hash(msghash, self.private_key)
 
         return MessageSignature(
             v=signed_msg.v,

tests/functional/geth/test_provider.py

@@ -11,7 +11,6 @@
 from web3 import AutoProvider, Web3
 from web3.exceptions import ContractLogicError as Web3ContractLogicError
 from web3.exceptions import ExtraDataLengthError
-from web3.middleware import geth_poa_middleware as ExtraDataToPOAMiddleware
 from web3.providers import HTTPProvider
 
 from ape.exceptions import (
@@ -26,6 +25,7 @@
     VirtualMachineError,
 )
 from ape.utils import to_int
+from ape.utils._web3_compat import ExtraDataToPOAMiddleware
 from ape_ethereum.ecosystem import Block
 from ape_ethereum.provider import DEFAULT_SETTINGS, EthereumNodeProvider
 from ape_ethereum.trace import TraceApproach

tests/functional/test_ecosystem.py

@@ -1188,11 +1188,11 @@ def get_calltree(self) -> CallTreeNode:
         {
             "name": "NumberChange",
             "calldata": {
-                "b": "0x3e..404b",
+                "b": "0x3ee0..404b",
                 "prevNum": 0,
                 "dynData": '"Dynamic"',
                 "newNum": 123,
-                "dynIndexed": "0x9f..a94d",
+                "dynIndexed": "0x9f3d..a94d",
             },
         }
     ]