feat: support request_header driven from configs (#2252)

docs/userguides/config.md

@@ -179,6 +179,39 @@ Install these plugins by running command:
 ape plugins install .
 ```
 
+## Request Headers
+
+For Ape's HTTP usage, such as requests made via `web3.py`, optionally specify extra request headers.
+
+```yaml
+request_headers:
+  # NOTE: Only using Content-Type as an example; can be any header key/value.
+  Content-Type: application/json
+```
+
+You can also specify request headers at the ecosystem, network, and provider levels:
+
+```yaml
+# NOTE: All the headers are the same only for demo purposes.
+# You can use headers you want for any of these config locations.
+ethereum:
+  # Apply to all requests made to ethereum networks.
+  request_headers:
+    Content-Type: application/json
+
+  mainnet:
+    # Apply to all requests made to ethereum:mainnet (using any provider)
+    request_headers:
+      Content-Type: application/json
+
+node:
+  # Apply to any request using the `node` provider.
+  request_headers:
+    Content-Type: application/json
+```
+
+To learn more about how request headers work in Ape, see [this section of the Networking guide](./networks.html#request-headers).
+
 ## Testing
 
 Configure your test accounts:

docs/userguides/networks.md

@@ -340,6 +340,43 @@ You may use one of:
 
 For the local network configuration, the default is `"max"`. Otherwise, it is `"auto"`.
 
+## Request Headers
+
+There are several layers of request-header configuration.
+They get merged into each-other in this order, with the exception being `User-Agent`, which has an append-behavior.
+
+- Default Ape headers (includes `User-Agent`)
+- Top-level configuration for headers (using `request_headers:` key)
+- Per-ecosystem configuration
+- Per-network configuration
+- Per-provider configuration
+
+Use the top-level `request_headers:` config to specify headers for every request.
+Use ecosystem-level specification for only requests made when connected to that ecosystem.
+Network and provider configurations work similarly; they are only used when connecting to that network or provider.
+
+Here is an example using each layer:
+
+```yaml
+request_headers:
+  Top-Level: "UseThisOnEveryRequest"
+
+ethereum:
+  request_headers:
+    Ecosystem-Level: "UseThisOnEveryEthereumRequest"
+  
+  mainnet:
+    request_headers:
+      Network-Level: "UseThisOnAllRequestsToEthereumMainnet"
+
+node:
+  request_headers:
+    Provider-Level: "UseThisOnAllRequestsUsingNodeProvider"
+```
+
+When using `User-Agent`, it will not override Ape's default `User-Agent` nor will each layer override each-other's.
+Instead, they are carefully appended to each other, allowing you to have a very customizable `User-Agent`.
+
 ## Local Network
 
 The default network in Ape is the local network (keyword `"local"`).

src/ape/api/config.py

@@ -333,6 +333,11 @@ def __init__(self, *args, **kwargs):
     The name of the project.
     """
 
+    request_headers: dict = {}
+    """
+    Extra request headers for all HTTP requests.
+    """
+
     version: str = ""
     """
     The version of the project.

src/ape/api/networks.py

@@ -13,6 +13,7 @@
 from eth_utils import keccak, to_int
 from ethpm_types import BaseModel, ContractType
 from ethpm_types.abi import ABIType, ConstructorABI, EventABI, MethodABI
+from pydantic import model_validator
 
 from ape.exceptions import (
     CustomError,
@@ -31,6 +32,7 @@
     ExtraAttributesMixin,
     ExtraModelAttributes,
     ManagerAccessMixin,
+    RPCHeaders,
     abstractmethod,
     cached_property,
     log_instead_of_fail,
@@ -68,7 +70,9 @@ class EcosystemAPI(ExtraAttributesMixin, BaseInterfaceModel):
     The name of the ecosystem. This should be set the same name as the plugin.
     """
 
-    request_header: dict
+    # TODO: In 0.9, make @property that returns value from config,
+    #   and use REQUEST_HEADER as plugin-defined constants.
+    request_header: dict = {}
     """A shareable HTTP header for network requests."""
 
     fee_token_symbol: str
@@ -80,6 +84,14 @@ class EcosystemAPI(ExtraAttributesMixin, BaseInterfaceModel):
     _default_network: Optional[str] = None
     """The default network of the ecosystem, such as ``local``."""
 
+    @model_validator(mode="after")
+    @classmethod
+    def _validate_ecosystem(cls, model):
+        headers = RPCHeaders(**model.request_header)
+        headers["User-Agent"] = f"ape-{model.name}"
+        model.request_header = dict(**headers)
+        return model
+
     @log_instead_of_fail(default="<EcosystemAPI>")
     def __repr__(self) -> str:
         return f"<{self.name}>"
@@ -289,9 +301,7 @@ def networks(self) -> dict[str, "NetworkAPI"]:
     @cached_property
     def _networks_from_plugins(self) -> dict[str, "NetworkAPI"]:
         return {
-            network_name: network_class(
-                name=network_name, ecosystem=self, request_header=self.request_header
-            )
+            network_name: network_class(name=network_name, ecosystem=self)
             for _, (ecosystem_name, network_name, network_class) in self.plugin_manager.networks
             if ecosystem_name == self.name
         }
@@ -647,6 +657,16 @@ def decode_custom_error(
             Optional[CustomError]: If it able to decode one, else ``None``.
         """
 
+    def _get_request_headers(self) -> RPCHeaders:
+        # Internal helper method called by NetworkManager
+        headers = RPCHeaders(**self.request_header)
+        # Have to do it this way to avoid "multiple-keys" error.
+        configured_headers: dict = self.config.get("request_headers", {})
+        for key, value in configured_headers.items():
+            headers[key] = value
+
+        return headers
+
 
 class ProviderContextManager(ManagerAccessMixin):
     """
@@ -809,7 +829,9 @@ class NetworkAPI(BaseInterfaceModel):
     ecosystem: EcosystemAPI
     """The ecosystem of the network."""
 
-    request_header: dict
+    # TODO: In 0.9, make @property that returns value from config,
+    #   and use REQUEST_HEADER as plugin-defined constants.
+    request_header: dict = {}
     """A shareable network HTTP header."""
 
     # See ``.default_provider`` which is the proper field.
@@ -1043,7 +1065,6 @@ def providers(self):  # -> dict[str, Partial[ProviderAPI]]
                     provider_class,
                     name=provider_name,
                     network=self,
-                    request_header=self.request_header,
                 )
 
         return providers
@@ -1285,6 +1306,16 @@ def verify_chain_id(self, chain_id: int):
         if self.name not in ("custom", LOCAL_NETWORK_NAME) and self.chain_id != chain_id:
             raise NetworkMismatchError(chain_id, self)
 
+    def _get_request_headers(self) -> RPCHeaders:
+        # Internal helper method called by NetworkManager
+        headers = RPCHeaders(**self.request_header)
+        # Have to do it this way to avoid multiple-keys error.
+        configured_headers: dict = self.config.get("request_headers", {})
+        for key, value in configured_headers.items():
+            headers[key] = value
+
+        return headers
+
 
 class ForkedNetworkAPI(NetworkAPI):
     @property

src/ape/api/providers.py

@@ -41,6 +41,7 @@
     log_instead_of_fail,
     raises_not_implemented,
 )
+from ape.utils.rpc import RPCHeaders
 
 if TYPE_CHECKING:
     from ape.api.accounts import TestAccountAPI
@@ -177,7 +178,9 @@ class ProviderAPI(BaseInterfaceModel):
     provider_settings: dict = {}
     """The settings for the provider, as overrides to the configuration."""
 
-    request_header: dict
+    # TODO: In 0.9, make @property that returns value from config,
+    #   and use REQUEST_HEADER as plugin-defined constants.
+    request_header: dict = {}
     """A header to set on HTTP/RPC requests."""
 
     block_page_size: int = 100
@@ -845,6 +848,16 @@ def get_virtual_machine_error(self, exception: Exception, **kwargs) -> VirtualMa
         """
         return VirtualMachineError(base_err=exception, **kwargs)
 
+    def _get_request_headers(self) -> RPCHeaders:
+        # Internal helper method called by NetworkManager
+        headers = RPCHeaders(**self.request_header)
+        # Have to do it this way to avoid "multiple-keys" error.
+        configured_headers: dict = self.config.get("request_headers", {})
+        for key, value in configured_headers.items():
+            headers[key] = value
+
+        return headers
+
 
 class TestProviderAPI(ProviderAPI):
     """

src/ape/managers/__init__.py

@@ -14,7 +14,7 @@
 
 ManagerAccessMixin.plugin_manager = PluginManager()
 ManagerAccessMixin.config_manager = ConfigManager(
-    request_header={"User-Agent": USER_AGENT},
+    request_header={"User-Agent": USER_AGENT, "Content-Type": "application/json"},
 )
 ManagerAccessMixin.compiler_manager = CompilerManager()
 ManagerAccessMixin.network_manager = NetworkManager()

src/ape/managers/config.py

@@ -17,6 +17,7 @@
     get_item_with_extras,
     only_raise_attribute_error,
 )
+from ape.utils.rpc import RPCHeaders
 
 CONFIG_FILE_NAME = "ape-config.yaml"
 
@@ -125,6 +126,14 @@ def isolate_data_folder(self) -> Iterator[Path]:
             finally:
                 self.DATA_FOLDER = original_data_folder
 
+    def _get_request_headers(self) -> RPCHeaders:
+        # Avoid multiple keys error by not initializing with both dicts.
+        headers = RPCHeaders(**self.REQUEST_HEADER)
+        for key, value in self.request_headers.items():
+            headers[key] = value
+
+        return headers
+
 
 def merge_configs(*cfgs: dict) -> dict:
     if len(cfgs) == 0:

src/ape/managers/networks.py

@@ -6,6 +6,7 @@
 from ape.api.networks import NetworkAPI
 from ape.exceptions import EcosystemNotFoundError, NetworkError, NetworkNotFoundError
 from ape.managers.base import BaseManager
+from ape.utils import RPCHeaders
 from ape.utils.basemodel import (
     ExtraAttributesMixin,
     ExtraModelAttributes,
@@ -85,6 +86,22 @@ def ecosystem(self) -> EcosystemAPI:
         """
         return self.network.ecosystem
 
+    def get_request_headers(
+        self, ecosystem_name: str, network_name: str, provider_name: str
+    ) -> RPCHeaders:
+        """
+        All request headers to be used when connecting to this network.
+        """
+        ecosystem = self.get_ecosystem(ecosystem_name)
+        network = ecosystem.get_network(network_name)
+        provider = network.get_provider(provider_name)
+        headers = self.config_manager._get_request_headers()
+        for obj in (ecosystem, network, provider):
+            for key, value in obj._get_request_headers().items():
+                headers[key] = value
+
+        return headers
+
     def fork(
         self,
         provider_name: Optional[str] = None,
@@ -225,12 +242,9 @@ def ecosystems(self) -> dict[str, EcosystemAPI]:
 
     @cached_property
     def _plugin_ecosystems(self) -> dict[str, EcosystemAPI]:
-        def to_kwargs(name: str) -> dict:
-            return {"name": name, "request_header": self.config_manager.REQUEST_HEADER}
-
         # Load plugins.
         plugins = self.plugin_manager.ecosystems
-        return {n: cls(**to_kwargs(n)) for n, cls in plugins}  # type: ignore[operator]
+        return {n: cls(name=n) for n, cls in plugins}  # type: ignore[operator]
 
     def create_custom_provider(
         self,
@@ -287,7 +301,6 @@ def create_custom_provider(
             network=network,
             provider_settings=provider_settings,
             data_folder=self.ethereum.data_folder / name,
-            request_header=network.request_header,
         )
 
     def __iter__(self) -> Iterator[str]:

src/ape/pytest/fixtures.py

@@ -16,7 +16,7 @@
 from ape.pytest.config import ConfigWrapper
 from ape.types import SnapshotID
 from ape.utils.basemodel import ManagerAccessMixin
-from ape.utils.misc import allow_disconnected
+from ape.utils.rpc import allow_disconnected
 
 
 class PytestApeFixtures(ManagerAccessMixin):

src/ape/utils/__init__.py

@@ -25,10 +25,8 @@
     DEFAULT_TRANSACTION_ACCEPTANCE_TIMEOUT,
     EMPTY_BYTES32,
     SOURCE_EXCLUDE_PATTERNS,
-    USER_AGENT,
     ZERO_ADDRESS,
     add_padding_to_strings,
-    allow_disconnected,
     as_our_module,
     cached_property,
     extract_nested_value,
@@ -44,7 +42,6 @@
     raises_not_implemented,
     run_until_complete,
     singledispatchmethod,
-    stream_response,
     to_int,
 )
 from ape.utils.os import (
@@ -61,6 +58,7 @@
     use_temp_sys_path,
 )
 from ape.utils.process import JoinableQueue, spawn
+from ape.utils.rpc import USER_AGENT, RPCHeaders, allow_disconnected, stream_response
 from ape.utils.testing import (
     DEFAULT_NUMBER_OF_TEST_ACCOUNTS,
     DEFAULT_TEST_ACCOUNT_BALANCE,
@@ -125,6 +123,7 @@
     "path_match",
     "raises_not_implemented",
     "returns_array",
+    "RPCHeaders",
     "run_in_tempdir",
     "run_until_complete",
     "singledispatchmethod",

src/ape/utils/_github.py

@@ -14,7 +14,8 @@
 
 from ape.exceptions import CompilerError, ProjectError, UnknownVersionError
 from ape.logging import logger
-from ape.utils.misc import USER_AGENT, cached_property, stream_response
+from ape.utils.misc import cached_property
+from ape.utils.rpc import USER_AGENT, stream_response
 
 
 class GitProcessWrapper: