User Agent 2.0, second attempt (boto#2977)

.changes/next-release/enhancement-Useragent-93485.json

@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "Useragent",
+  "description": "Update User-Agent header format"
+}

botocore/args.py

@@ -28,6 +28,7 @@
 from botocore.regions import EndpointResolverBuiltins as EPRBuiltins
 from botocore.regions import EndpointRulesetResolver
 from botocore.signers import RequestSigner
+from botocore.useragent import UserAgentString
 from botocore.utils import ensure_boolean, is_s3_accelerate_url
 
 logger = logging.getLogger(__name__)
@@ -55,6 +56,9 @@
     'us-west-1',
     'us-west-2',
 ]
+# Maximum allowed length of the ``user_agent_appid`` config field. Longer
+# values result in a warning-level log message.
+USERAGENT_APPID_MAXLEN = 50
 
 
 class ClientArgsCreator:
@@ -66,13 +70,17 @@ def __init__(
         loader,
         exceptions_factory,
         config_store,
+        user_agent_creator=None,
     ):
         self._event_emitter = event_emitter
-        self._user_agent = user_agent
         self._response_parser_factory = response_parser_factory
         self._loader = loader
         self._exceptions_factory = exceptions_factory
         self._config_store = config_store
+        if user_agent_creator is None:
+            self._session_ua_creator = UserAgentString.from_environment()
+        else:
+            self._session_ua_creator = user_agent_creator
 
     def get_client_args(
         self,
@@ -159,6 +167,13 @@ def get_client_args(
             event_emitter,
         )
 
+        # Copy the session's user agent factory and adds client configuration.
+        client_ua_creator = self._session_ua_creator.with_client_config(
+            new_config
+        )
+        supplied_ua = client_config.user_agent if client_config else None
+        new_config._supplied_user_agent = supplied_ua
+
         return {
             'serializer': serializer,
             'endpoint': endpoint,
@@ -171,6 +186,7 @@ def get_client_args(
             'partition': partition,
             'exceptions_factory': self._exceptions_factory,
             'endpoint_ruleset_resolver': ruleset_resolver,
+            'user_agent_creator': client_ua_creator,
         }
 
     def compute_client_args(
@@ -193,14 +209,6 @@ def compute_client_args(
             if raw_value is not None:
                 parameter_validation = ensure_boolean(raw_value)
 
-        # Override the user agent if specified in the client config.
-        user_agent = self._user_agent
-        if client_config is not None:
-            if client_config.user_agent is not None:
-                user_agent = client_config.user_agent
-            if client_config.user_agent_extra is not None:
-                user_agent += ' %s' % client_config.user_agent_extra
-
         s3_config = self.compute_s3_config(client_config)
         endpoint_config = self._compute_endpoint_config(
             service_name=service_name,
@@ -211,13 +219,23 @@ def compute_client_args(
             s3_config=s3_config,
         )
         endpoint_variant_tags = endpoint_config['metadata'].get('tags', [])
+
+        # Some third-party libraries expect the final user-agent string in
+        # ``client.meta.config.user_agent``. To maintain backwards
+        # compatibility, the preliminary user-agent string (before any Config
+        # object modifications and without request-specific user-agent
+        # components) is stored in the new Config object's ``user_agent``
+        # property but not used by Botocore itself.
+        preliminary_ua_string = self._session_ua_creator.with_client_config(
+            client_config
+        ).to_string()
         # Create a new client config to be passed to the client based
         # on the final values. We do not want the user to be able
         # to try to modify an existing client with a client config.
         config_kwargs = dict(
             region_name=endpoint_config['region_name'],
             signature_version=endpoint_config['signature_version'],
-            user_agent=user_agent,
+            user_agent=preliminary_ua_string,
         )
         if 'dualstack' in endpoint_variant_tags:
             config_kwargs.update(use_dualstack_endpoint=True)
@@ -234,9 +252,12 @@ def compute_client_args(
                 client_cert=client_config.client_cert,
                 inject_host_prefix=client_config.inject_host_prefix,
                 tcp_keepalive=client_config.tcp_keepalive,
+                user_agent_extra=client_config.user_agent_extra,
+                user_agent_appid=client_config.user_agent_appid,
             )
         self._compute_retry_config(config_kwargs)
         self._compute_connect_timeout(config_kwargs)
+        self._compute_user_agent_appid_config(config_kwargs)
         s3_config = self.compute_s3_config(client_config)
 
         is_s3_service = self._is_s3_service(service_name)
@@ -249,7 +270,6 @@ def compute_client_args(
         return {
             'service_name': service_name,
             'parameter_validation': parameter_validation,
-            'user_agent': user_agent,
             'endpoint_config': endpoint_config,
             'protocol': protocol,
             'config_kwargs': config_kwargs,
@@ -646,3 +666,19 @@ def compute_endpoint_resolver_builtin_defaults(
             ),
             EPRBuiltins.SDK_ENDPOINT: given_endpoint,
         }
+
+    def _compute_user_agent_appid_config(self, config_kwargs):
+        user_agent_appid = config_kwargs.get('user_agent_appid')
+        if user_agent_appid is None:
+            user_agent_appid = self._config_store.get_config_variable(
+                'user_agent_appid'
+            )
+        if (
+            user_agent_appid is not None
+            and len(user_agent_appid) > USERAGENT_APPID_MAXLEN
+        ):
+            logger.warning(
+                'The configured value for user_agent_appid exceeds the '
+                f'maximum length of {USERAGENT_APPID_MAXLEN} characters.'
+            )
+        config_kwargs['user_agent_appid'] = user_agent_appid

botocore/client.py

@@ -39,6 +39,7 @@
 from botocore.model import ServiceModel
 from botocore.paginate import Paginator
 from botocore.retries import adaptive, standard
+from botocore.useragent import UserAgentString
 from botocore.utils import (
     CachedProperty,
     EventbridgeSignerSetter,
@@ -91,6 +92,7 @@ def __init__(
         response_parser_factory=None,
         exceptions_factory=None,
         config_store=None,
+        user_agent_creator=None,
     ):
         self._loader = loader
         self._endpoint_resolver = endpoint_resolver
@@ -105,6 +107,7 @@ def __init__(
         # config and environment variables (and potentially more in the
         # future).
         self._config_store = config_store
+        self._user_agent_creator = user_agent_creator
 
     def create_client(
         self,
@@ -481,6 +484,7 @@ def _get_client_args(
             self._loader,
             self._exceptions_factory,
             config_store=self._config_store,
+            user_agent_creator=self._user_agent_creator,
         )
         return args_creator.get_client_args(
             service_model,
@@ -840,6 +844,7 @@ def __init__(
         partition,
         exceptions_factory,
         endpoint_ruleset_resolver=None,
+        user_agent_creator=None,
     ):
         self._serializer = serializer
         self._endpoint = endpoint
@@ -859,6 +864,13 @@ def __init__(
         )
         self._exceptions_factory = exceptions_factory
         self._exceptions = None
+        self._user_agent_creator = user_agent_creator
+        if self._user_agent_creator is None:
+            self._user_agent_creator = (
+                UserAgentString.from_environment().with_client_config(
+                    self._client_config
+                )
+            )
         self._register_handlers()
 
     def __getattr__(self, item):
@@ -996,7 +1008,7 @@ def _convert_to_request_dict(
         if headers is not None:
             request_dict['headers'].update(headers)
         if set_user_agent_header:
-            user_agent = self._client_config.user_agent
+            user_agent = self._user_agent_creator.to_string()
         else:
             user_agent = None
         prepare_request_dict(

botocore/config.py

@@ -38,6 +38,12 @@ class Config:
     :param user_agent_extra: The value to append to the current User-Agent
         header value.
 
+    :type user_agent_appid: str
+    :param user_agent_appid: A value that gets included in the User-Agent
+        string in the format "app/<user_agent_appid>". Allowed characters are
+        ASCII alphanumerics and ``!$%&'*+-.^_`|~``. All other characters will
+        be replaced by a ``-``.
+
     :type connect_timeout: float or int
     :param connect_timeout: The time in seconds till a timeout exception is
         thrown when attempting to make a connection. The default is 60
@@ -201,6 +207,7 @@ class Config:
             ('signature_version', None),
             ('user_agent', None),
             ('user_agent_extra', None),
+            ('user_agent_appid', None),
             ('connect_timeout', DEFAULT_TIMEOUT),
             ('read_timeout', DEFAULT_TIMEOUT),
             ('parameter_validation', True),

botocore/configprovider.py

@@ -139,6 +139,7 @@
     # We can't have a default here for v1 because we need to defer to
     # whatever the defaults are in _retry.json.
     'max_attempts': ('max_attempts', 'AWS_MAX_ATTEMPTS', None, int),
+    'user_agent_appid': ('sdk_ua_app_id', 'AWS_SDK_UA_APP_ID', None, None),
 }
 # A mapping for the s3 specific configuration vars. These are the configuration
 # vars that typically go in the s3 section of the config file. This mapping

botocore/session.py

@@ -64,12 +64,16 @@
 from botocore.model import ServiceModel
 from botocore.parsers import ResponseParserFactory
 from botocore.regions import EndpointResolver
+from botocore.useragent import UserAgentString
 from botocore.utils import (
     EVENT_ALIASES,
     IMDSRegionProvider,
     validate_region_name,
 )
 
+from botocore.compat import HAS_CRT  # noqa
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -165,6 +169,7 @@ def _register_components(self):
         self._register_monitor()
         self._register_default_config_resolver()
         self._register_smart_defaults_factory()
+        self._register_user_agent_creator()
 
     def _register_event_emitter(self):
         self._components.register_component('event_emitter', self._events)
@@ -263,6 +268,10 @@ def _register_monitor(self):
             'monitor', self._create_csm_monitor
         )
 
+    def _register_user_agent_creator(self):
+        uas = UserAgentString.from_environment()
+        self._components.register_component('user_agent_creator', uas)
+
     def _create_csm_monitor(self):
         if self.get_config_variable('csm_enabled'):
             client_id = self.get_config_variable('csm_client_id')
@@ -283,12 +292,8 @@ def _create_csm_monitor(self):
         return None
 
     def _get_crt_version(self):
-        try:
-            import awscrt
-
-            return awscrt.__version__
-        except AttributeError:
-            return "Unknown"
+        user_agent_creator = self.get_component('user_agent_creator')
+        return user_agent_creator._crt_version or 'Unknown'
 
     @property
     def available_profiles(self):
@@ -953,6 +958,15 @@ def create_client(
         endpoint_resolver = self._get_internal_component('endpoint_resolver')
         exceptions_factory = self._get_internal_component('exceptions_factory')
         config_store = self.get_component('config_store')
+        user_agent_creator = self.get_component('user_agent_creator')
+        # Session configuration values for the user agent string are applied
+        # just before each client creation because they may have been modified
+        # at any time between session creation and client creation.
+        user_agent_creator.set_session_config(
+            session_user_agent_name=self.user_agent_name,
+            session_user_agent_version=self.user_agent_version,
+            session_user_agent_extra=self.user_agent_extra,
+        )
         defaults_mode = self._resolve_defaults_mode(config, config_store)
         if defaults_mode != 'legacy':
             smart_defaults_factory = self._get_internal_component(
@@ -972,6 +986,7 @@ def create_client(
             response_parser_factory,
             exceptions_factory,
             config_store,
+            user_agent_creator=user_agent_creator,
         )
         client = client_creator.create_client(
             service_name=service_name,