Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Account ID endpoint routing #3031

Closed
wants to merge 29 commits into from
Closed
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
f22ca61
add `account_id` to Credentials
Oct 4, 2023
bc166c0
unit tests for `account_id` in credentials
Oct 4, 2023
958dc8e
add `account_id_endpoint_mode` config setting to endpoint resolution
Oct 4, 2023
aa604a3
unit tests for `account_id_endpoint_mode`
Oct 4, 2023
78f4a37
shorten test name
Oct 4, 2023
e4188ba
add `account_id` to set_credentials
Oct 4, 2023
c79dc03
pr feedback
Oct 5, 2023
9bc1a2e
clean up config docs
Oct 5, 2023
dd4d7c3
pr feedback
Oct 6, 2023
109dfba
only include account_id in credential fetchers if successfully parsed
Oct 6, 2023
24f6df2
formatting fixes
Oct 6, 2023
e321aeb
updates to test_endpoint_provider.py
Oct 10, 2023
f41ff9c
changelog
Oct 11, 2023
a183e91
pr feedback
Oct 13, 2023
6892215
string formatting
Oct 13, 2023
c38ebf4
log message formatting
Oct 17, 2023
ef2397b
pass `account_id_endpoint_mode` as an input param to resolver and sim…
Oct 18, 2023
49e071f
clean up _resolve_account_id_builtin
Oct 19, 2023
dee1e9f
move account id resolution to separate class
Oct 26, 2023
83017c5
bug fix. account id may not always be a builtin when configured as a …
Oct 27, 2023
9db7688
add back line lost in rebase
Oct 30, 2023
d4b00a6
pr feedback. pass builtin resolver to ruleset resolver and refactor s…
Nov 1, 2023
747c59c
error tests
Nov 1, 2023
e94c438
add higher level builtin resolver
Nov 2, 2023
b7088d6
fix quotes
Nov 3, 2023
9d3deab
change variable name
Nov 3, 2023
c6b2d0f
split account ID refresh into two tests
Nov 3, 2023
9febc3c
add creds method to message
Nov 3, 2023
eccd175
pr feedback
Nov 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changes/next-release/feature-endpoints-57979.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"type": "feature",
"category": "endpoints",
"description": "Adds support for account ID endpoint routing"
}
32 changes: 31 additions & 1 deletion botocore/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import botocore.serialize
from botocore.config import Config
from botocore.endpoint import EndpointCreator
from botocore.regions import CredentialBuiltinResolver, EndpointBuiltinResolver
from botocore.regions import EndpointResolverBuiltins as EPRBuiltins
from botocore.regions import EndpointRulesetResolver
from botocore.signers import RequestSigner
Expand Down Expand Up @@ -152,7 +153,9 @@ def get_client_args(
protocol, parameter_validation
)
response_parser = botocore.parsers.create_parser(protocol)

builtin_resolver = self._construct_builtin_resolver(
credentials, new_config
)
ruleset_resolver = self._build_endpoint_resolver(
endpoints_ruleset_data,
partition_data,
Expand All @@ -165,6 +168,7 @@ def get_client_args(
is_secure,
endpoint_bridge,
event_emitter,
builtin_resolver,
)

# Copy the session's user agent factory and adds client configuration.
Expand Down Expand Up @@ -267,11 +271,15 @@ def compute_client_args(
client_config.disable_request_compression
),
client_context_params=client_config.client_context_params,
account_id_endpoint_mode=(
client_config.account_id_endpoint_mode
),
)
self._compute_retry_config(config_kwargs)
self._compute_connect_timeout(config_kwargs)
self._compute_user_agent_appid_config(config_kwargs)
self._compute_request_compression_config(config_kwargs)
self._compute_account_id_endpoint_mode(config_kwargs)
s3_config = self.compute_s3_config(client_config)

is_s3_service = self._is_s3_service(service_name)
Expand Down Expand Up @@ -598,12 +606,29 @@ def _validate_min_compression_size(self, min_size):

return min_size

def _compute_account_id_endpoint_mode(self, config_kwargs):
ep_mode = config_kwargs.get('account_id_endpoint_mode')
if ep_mode is None:
ep_mode = self._config_store.get_config_variable(
'account_id_endpoint_mode'
)
config_kwargs['account_id_endpoint_mode'] = ep_mode

def _ensure_boolean(self, val):
if isinstance(val, bool):
return val
else:
return val.lower() == 'true'

def _construct_builtin_resolver(self, credentials, client_config):
credential_builtin_resolver = CredentialBuiltinResolver(
credentials, client_config.account_id_endpoint_mode
)
# This only includes CredentialBuiltinResolver for now, but may grow
# as more endpoint builtins are added.
dlm6693 marked this conversation as resolved.
Show resolved Hide resolved
resolver_map = {'credentials': credential_builtin_resolver}
return EndpointBuiltinResolver(resolver_map)

def _build_endpoint_resolver(
self,
endpoints_ruleset_data,
Expand All @@ -617,6 +642,7 @@ def _build_endpoint_resolver(
is_secure,
endpoint_bridge,
event_emitter,
builtin_resolver,
):
if endpoints_ruleset_data is None:
return None
Expand Down Expand Up @@ -666,6 +692,7 @@ def _build_endpoint_resolver(
event_emitter=event_emitter,
use_ssl=is_secure,
requested_auth_scheme=sig_version,
builtin_resolver=builtin_resolver,
)

def compute_endpoint_resolver_builtin_defaults(
Expand Down Expand Up @@ -750,6 +777,9 @@ def compute_endpoint_resolver_builtin_defaults(
's3_disable_multiregion_access_points', False
),
EPRBuiltins.SDK_ENDPOINT: given_endpoint,
# account ID is calculated later if account based routing is
# enabled and configured for the service
EPRBuiltins.AWS_ACCOUNT_ID: None,
}

def _compute_user_agent_appid_config(self, config_kwargs):
Expand Down
13 changes: 12 additions & 1 deletion botocore/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,17 @@ class Config:
documentation. Invalid parameters or ones that are not used by the
specified service will be ignored.

Defaults to None.
:type account_id_endpoint_mode: str
:param account_id_endpoint_mode: Set account ID based endpoint behavior
for supported operations.
Valid options are:

* ``preferred`` -- Use account ID based endpoint routing if available.
* ``required`` -- Raise ``AccountIdNotFound`` exception if account ID
based endpoint routing is unavailable.
* ``disabled`` -- Disable account ID based endpoint routing.

If not specified, the default behavior is ``preferred``.
"""

OPTION_DEFAULTS = OrderedDict(
Expand Down Expand Up @@ -257,6 +267,7 @@ class Config:
('request_min_compression_size_bytes', None),
('disable_request_compression', None),
('client_context_params', None),
('account_id_endpoint_mode', None),
]
)

Expand Down
6 changes: 6 additions & 0 deletions botocore/configprovider.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,12 @@
False,
utils.ensure_boolean,
),
'account_id_endpoint_mode': (
'account_id_endpoint_mode',
'AWS_ACCOUNT_ID_ENDPOINT_MODE',
'preferred',
None,
),
}
# A mapping for the s3 specific configuration vars. These are the configuration
# vars that typically go in the s3 section of the config file. This mapping
Expand Down
Loading