Workaround botocore bug in S3 URL Handler backend (#19056)

See boto/botocore#2948 for the upstream bug fix.

This only occurs for toekn-based auth, and therefore wasn't caught in
tests or in local testing.

src/python/pants/backend/url_handlers/s3/integration_test.py

@@ -45,7 +45,7 @@ def do_patching(expected_url):
         botocore = SimpleNamespace()
         botocore.exceptions = SimpleNamespace(NoCredentialsError=Exception)
         fake_session = object()
-        fake_creds = SimpleNamespace(access_key="ACCESS", secret_key="SECRET")
+        fake_creds = SimpleNamespace(access_key="ACCESS", secret_key="SECRET", token=None)
         botocore.session = SimpleNamespace(get_session=lambda: fake_session)
 
         def fake_resolver_creator(session):

src/python/pants/backend/url_handlers/s3/register.py

@@ -1,5 +1,7 @@
 # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
+from __future__ import annotations
+
 import logging
 from dataclasses import dataclass
 from types import SimpleNamespace
@@ -72,9 +74,14 @@ async def download_from_s3(request: S3DownloadFile, aws_credentials: AWSCredenti
     if request.query:
         path_style_url += f"?{request.query}"
 
+    headers: dict[str, str] = {}
+    if aws_credentials.creds.token:
+        # Workaround https://github.com/boto/botocore/pull/2948
+        headers["X-Amz-Security-Token"] = ""
+
     http_request = SimpleNamespace(
         url=path_style_url,
-        headers={},
+        headers=headers,
         method="GET",
         auth_path=None,
     )