From 2c9f20f7434baf042eda44d2e34bbb609ea19e14 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 3 Jul 2017 12:58:39 -0700 Subject: [PATCH 01/10] add support for latest aiohttp/botocore --- aiobotocore/args.py | 18 ++++++++++++++++-- requirements-dev.txt | 4 ++-- setup.py | 2 +- tests/test_patches.py | 9 +++++---- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/aiobotocore/args.py b/aiobotocore/args.py index ebf9e624..7fd842b7 100644 --- a/aiobotocore/args.py +++ b/aiobotocore/args.py @@ -4,10 +4,14 @@ import botocore.serialize import botocore.parsers from botocore.signers import RequestSigner +from packaging.version import parse as parse_version from .config import AioConfig from .endpoint import AioEndpointCreator +# https://github.com/boto/botocore/commit/5af85bf0b3cf6dac8d0487a2e97425cf3d53e4ee +BOTO_1_5_71 = parse_version(botocore.__version__) >= parse_version('1.5.71') + class AioClientArgsCreator(botocore.args.ClientArgsCreator): def __init__(self, event_emitter, user_agent, response_parser_factory, @@ -16,6 +20,8 @@ def __init__(self, event_emitter, user_agent, response_parser_factory, response_parser_factory, loader, exceptions_factory) self._loop = loop or asyncio.get_event_loop() + # NOTE: we override this so we can pull out the custom params from AioConfig + # and use an AioEndpointCreator def get_client_args(self, service_model, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, client_config, endpoint_bridge): @@ -31,9 +37,17 @@ def get_client_args(self, service_model, region_name, is_secure, s3_config = final_args['s3_config'] partition = endpoint_config['metadata'].get('partition', None) + signing_region = endpoint_config['signing_region'] + endpoint_region_name = endpoint_config['region_name'] + if BOTO_1_5_71: + if signing_region is None and endpoint_region_name is None: + signing_region, endpoint_region_name = \ + self._get_default_s3_region(service_name, endpoint_bridge) + config_kwargs['region_name'] = endpoint_region_name + event_emitter = copy.copy(self._event_emitter) signer = RequestSigner( - service_name, endpoint_config['signing_region'], + service_name, signing_region, endpoint_config['signing_name'], endpoint_config['signature_version'], credentials, event_emitter) @@ -49,7 +63,7 @@ def get_client_args(self, service_model, region_name, is_secure, endpoint_creator = AioEndpointCreator(event_emitter, self._loop) endpoint = endpoint_creator.create_endpoint( - service_model, region_name=endpoint_config['region_name'], + service_model, region_name=endpoint_region_name, endpoint_url=endpoint_config['endpoint_url'], verify=verify, response_parser_factory=self._response_parser_factory, max_pool_connections=new_config.max_pool_connections, diff --git a/requirements-dev.txt b/requirements-dev.txt index 2db5a121..7e075e5f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,8 +5,8 @@ moto==0.4.31 pytest-cov==2.5.1 pytest==3.1.2 sphinx==1.6.3 -aiohttp==2.1.0 -botocore==1.5.70 +aiohttp==2.2.2 +botocore==1.5.78 multidict==3.1.0 wrapt==1.10.10 dill==0.2.6 \ No newline at end of file diff --git a/setup.py b/setup.py index 1cad43f8..5ef68f71 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # aiohttp requirement is pegged as we need to manually ensure that # https://github.com/aio-libs/aiobotocore/pull/248 will continue working -install_requires = ['botocore>=1.5.34, <=1.5.70', 'aiohttp>=2.0.4, <=2.2.0', +install_requires = ['botocore>=1.5.34, <=1.5.78', 'aiohttp>=2.0.4, <=2.2.2', 'multidict>=2.1.4', 'wrapt>=1.10.10', 'packaging>=16.8'] PY_VER = sys.version_info diff --git a/tests/test_patches.py b/tests/test_patches.py index 106b3eb1..7072bcaa 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -35,14 +35,15 @@ # for our patch of _wait StreamReader: {'dc5a5a33e86cedf2d2c8413d951d2274a79303f4'}, - # for digging into _protocol - ClientResponse: {'d1e0c16dea4fe3426caa1e9b0dc9f5f1992d838e'}, + # for digging into _protocol ( 2.1.x, 2.2.x ) + ClientResponse: {'d1e0c16dea4fe3426caa1e9b0dc9f5f1992d838e', 'bc374038ac3bfd7cc13dadb6aebbf0f67ebb7620'}, } # These are guards to our main patches _API_DIGESTS = { - ClientArgsCreator: {'60b8b70358d25dcce1ad74b2f5f0b4830d5749a3'}, - ClientCreator: {'a8c12a6b933fa22477cd4d168a96443be002ae79'}, + # ( <=1.5.70, >=1.5.71) + ClientArgsCreator: {'60b8b70358d25dcce1ad74b2f5f0b4830d5749a3', '813098fb076fc4739f7f6ec335960845e119f5f4'}, + ClientCreator: {'a8c12a6b933fa22477cd4d168a96443be002ae79', '1533fe9ffa395f8555ffad0fb5b6f5294d2c0f30'}, BaseClient: {'a91ffdb8d0c7cb2dfd63a4332a0a7a76e76cef28'}, Config: {'c6bdc8f47c90e114d406ecab3fcfbc6e4d034279'}, convert_to_response_dict: {'ed634b3f0c24f8858aee8ed745051397270b1e46'}, From 76a1ec58624d64c15b450dfc2e72269a04487803 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 3 Jul 2017 13:00:04 -0700 Subject: [PATCH 02/10] bump version --- CHANGES.txt | 3 ++- aiobotocore/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 307ac8b5..6d366c77 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,8 @@ Changes ------- -0.4.2a0 (XXXX-XX-XX) +0.4.2 (2017-07-03) ^^^^^^^^^^^^^^^^^^ +* update supported aiohttp/botocore versions 0.4.1 (2017-06-27) ^^^^^^^^^^^^^^^^^^ diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index 14ed0a33..33a72b80 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1,4 +1,4 @@ from .session import get_session, AioSession __all__ = ['get_session', 'AioSession'] -__version__ = '0.4.2a0' +__version__ = '0.4.2' From 2511c7112e2ecc433c8346a417f8274c6493f7c9 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 3 Jul 2017 13:08:13 -0700 Subject: [PATCH 03/10] pep fixes --- aiobotocore/args.py | 4 ++-- tests/test_patches.py | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/aiobotocore/args.py b/aiobotocore/args.py index 7fd842b7..0a747597 100644 --- a/aiobotocore/args.py +++ b/aiobotocore/args.py @@ -20,8 +20,8 @@ def __init__(self, event_emitter, user_agent, response_parser_factory, response_parser_factory, loader, exceptions_factory) self._loop = loop or asyncio.get_event_loop() - # NOTE: we override this so we can pull out the custom params from AioConfig - # and use an AioEndpointCreator + # NOTE: we override this so we can pull out the custom AioConfig params and + # use an AioEndpointCreator def get_client_args(self, service_model, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, client_config, endpoint_bridge): diff --git a/tests/test_patches.py b/tests/test_patches.py index 7072bcaa..a26d79b2 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -36,14 +36,17 @@ StreamReader: {'dc5a5a33e86cedf2d2c8413d951d2274a79303f4'}, # for digging into _protocol ( 2.1.x, 2.2.x ) - ClientResponse: {'d1e0c16dea4fe3426caa1e9b0dc9f5f1992d838e', 'bc374038ac3bfd7cc13dadb6aebbf0f67ebb7620'}, + ClientResponse: {'d1e0c16dea4fe3426caa1e9b0dc9f5f1992d838e', + 'bc374038ac3bfd7cc13dadb6aebbf0f67ebb7620'}, } # These are guards to our main patches _API_DIGESTS = { # ( <=1.5.70, >=1.5.71) - ClientArgsCreator: {'60b8b70358d25dcce1ad74b2f5f0b4830d5749a3', '813098fb076fc4739f7f6ec335960845e119f5f4'}, - ClientCreator: {'a8c12a6b933fa22477cd4d168a96443be002ae79', '1533fe9ffa395f8555ffad0fb5b6f5294d2c0f30'}, + ClientArgsCreator: {'60b8b70358d25dcce1ad74b2f5f0b4830d5749a3', + '813098fb076fc4739f7f6ec335960845e119f5f4'}, + ClientCreator: {'a8c12a6b933fa22477cd4d168a96443be002ae79', + '1533fe9ffa395f8555ffad0fb5b6f5294d2c0f30'}, BaseClient: {'a91ffdb8d0c7cb2dfd63a4332a0a7a76e76cef28'}, Config: {'c6bdc8f47c90e114d406ecab3fcfbc6e4d034279'}, convert_to_response_dict: {'ed634b3f0c24f8858aee8ed745051397270b1e46'}, From be6d551125eb35ab81e2e2eec8ae7fbd786108ed Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 3 Jul 2017 13:26:06 -0700 Subject: [PATCH 04/10] workaround for bad releases --- tests/conftest.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 634046d3..a4f30113 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,15 @@ from aiobotocore.config import AioConfig import tempfile import shutil +import sys +from packaging.version import parse as parse_version + + +PY_34 = (3, 4) <= sys.version_info <= (3, 5) + +AIOHTTP_BAD_CLOSE_VERSION = PY_34 and (parse_version('2.2.0') <= + parse_version(aiohttp.__version__) <= + parse_version('2.2.2')) @pytest.fixture(scope="session", params=[True, False], @@ -243,7 +252,10 @@ def f(): client = loop.run_until_complete(f()) def fin(): - loop.run_until_complete(client.close()) + if AIOHTTP_BAD_CLOSE_VERSION: + client.close() + else: + loop.run_until_complete(client.close()) request.addfinalizer(fin) return client From 336d13fa0f89bd3059d0270296beefc82f9b4f7c Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 3 Jul 2017 13:54:45 -0700 Subject: [PATCH 05/10] better fix --- tests/conftest.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a4f30113..4af07874 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,13 @@ parse_version('2.2.2')) +if AIOHTTP_BAD_CLOSE_VERSION: + from asyncio import coroutines + import aiohttp.helpers + + coroutines._COROUTINE_TYPES += (aiohttp.helpers._CoroGuard, ) + + @pytest.fixture(scope="session", params=[True, False], ids=['debug[true]', 'debug[false]']) def debug(request): @@ -252,10 +259,7 @@ def f(): client = loop.run_until_complete(f()) def fin(): - if AIOHTTP_BAD_CLOSE_VERSION: - client.close() - else: - loop.run_until_complete(client.close()) + loop.run_until_complete(client.close()) request.addfinalizer(fin) return client From dc14662b3a8471fd223505b265941502248868c2 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 3 Jul 2017 14:42:00 -0700 Subject: [PATCH 06/10] simplifying change --- aiobotocore/args.py | 13 ++++--------- setup.py | 2 +- tests/test_patches.py | 4 +--- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/aiobotocore/args.py b/aiobotocore/args.py index 0a747597..e509a4c1 100644 --- a/aiobotocore/args.py +++ b/aiobotocore/args.py @@ -4,14 +4,10 @@ import botocore.serialize import botocore.parsers from botocore.signers import RequestSigner -from packaging.version import parse as parse_version from .config import AioConfig from .endpoint import AioEndpointCreator -# https://github.com/boto/botocore/commit/5af85bf0b3cf6dac8d0487a2e97425cf3d53e4ee -BOTO_1_5_71 = parse_version(botocore.__version__) >= parse_version('1.5.71') - class AioClientArgsCreator(botocore.args.ClientArgsCreator): def __init__(self, event_emitter, user_agent, response_parser_factory, @@ -39,11 +35,10 @@ def get_client_args(self, service_model, region_name, is_secure, signing_region = endpoint_config['signing_region'] endpoint_region_name = endpoint_config['region_name'] - if BOTO_1_5_71: - if signing_region is None and endpoint_region_name is None: - signing_region, endpoint_region_name = \ - self._get_default_s3_region(service_name, endpoint_bridge) - config_kwargs['region_name'] = endpoint_region_name + if signing_region is None and endpoint_region_name is None: + signing_region, endpoint_region_name = \ + self._get_default_s3_region(service_name, endpoint_bridge) + config_kwargs['region_name'] = endpoint_region_name event_emitter = copy.copy(self._event_emitter) signer = RequestSigner( diff --git a/setup.py b/setup.py index 5ef68f71..63e77526 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # aiohttp requirement is pegged as we need to manually ensure that # https://github.com/aio-libs/aiobotocore/pull/248 will continue working -install_requires = ['botocore>=1.5.34, <=1.5.78', 'aiohttp>=2.0.4, <=2.2.2', +install_requires = ['botocore>=1.5.71, <=1.5.78', 'aiohttp>=2.0.4, <=2.2.2', 'multidict>=2.1.4', 'wrapt>=1.10.10', 'packaging>=16.8'] PY_VER = sys.version_info diff --git a/tests/test_patches.py b/tests/test_patches.py index a26d79b2..4f2da355 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -42,9 +42,7 @@ # These are guards to our main patches _API_DIGESTS = { - # ( <=1.5.70, >=1.5.71) - ClientArgsCreator: {'60b8b70358d25dcce1ad74b2f5f0b4830d5749a3', - '813098fb076fc4739f7f6ec335960845e119f5f4'}, + ClientArgsCreator: {'813098fb076fc4739f7f6ec335960845e119f5f4'}, ClientCreator: {'a8c12a6b933fa22477cd4d168a96443be002ae79', '1533fe9ffa395f8555ffad0fb5b6f5294d2c0f30'}, BaseClient: {'a91ffdb8d0c7cb2dfd63a4332a0a7a76e76cef28'}, From 0f7ebfa7c4e28606a3b84b4e36ce8f6e445c6a90 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 3 Jul 2017 14:44:14 -0700 Subject: [PATCH 07/10] tweak changes --- CHANGES.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6d366c77..93aea50e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,7 +2,8 @@ Changes ------- 0.4.2 (2017-07-03) ^^^^^^^^^^^^^^^^^^ -* update supported aiohttp/botocore versions +* update supported aiohttp requirement to: >=2.0.4, <=2.2.2 +* update supported botocore requirement to: >=1.5.71, <=1.5.78 0.4.1 (2017-06-27) ^^^^^^^^^^^^^^^^^^ From 1a34216bb2d2c275c57f35ec3bbec8c340617464 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 14:32:42 -0700 Subject: [PATCH 08/10] bump req --- CHANGES.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 93aea50e..f06d32d0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,7 +2,7 @@ Changes ------- 0.4.2 (2017-07-03) ^^^^^^^^^^^^^^^^^^ -* update supported aiohttp requirement to: >=2.0.4, <=2.2.2 +* update supported aiohttp requirement to: >=2.0.4, <=2.3.0 * update supported botocore requirement to: >=1.5.71, <=1.5.78 0.4.1 (2017-06-27) diff --git a/setup.py b/setup.py index 63e77526..ee387008 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # aiohttp requirement is pegged as we need to manually ensure that # https://github.com/aio-libs/aiobotocore/pull/248 will continue working -install_requires = ['botocore>=1.5.71, <=1.5.78', 'aiohttp>=2.0.4, <=2.2.2', +install_requires = ['botocore>=1.5.71, <=1.5.78', 'aiohttp>=2.0.4, <=2.3.0', 'multidict>=2.1.4', 'wrapt>=1.10.10', 'packaging>=16.8'] PY_VER = sys.version_info From 1bd5dc937a43c3434c75b7c5044db215be8dae0d Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 14:54:02 -0700 Subject: [PATCH 09/10] remove monkeypatch --- tests/conftest.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4af07874..c250ee14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,22 +7,10 @@ import tempfile import shutil import sys -from packaging.version import parse as parse_version PY_34 = (3, 4) <= sys.version_info <= (3, 5) -AIOHTTP_BAD_CLOSE_VERSION = PY_34 and (parse_version('2.2.0') <= - parse_version(aiohttp.__version__) <= - parse_version('2.2.2')) - - -if AIOHTTP_BAD_CLOSE_VERSION: - from asyncio import coroutines - import aiohttp.helpers - - coroutines._COROUTINE_TYPES += (aiohttp.helpers._CoroGuard, ) - @pytest.fixture(scope="session", params=[True, False], ids=['debug[true]', 'debug[false]']) From 3f17b375f2e2d7b3341d583294b04bd0e8245d66 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 16:00:34 -0700 Subject: [PATCH 10/10] bugfix --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7e075e5f..cee67bfb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ moto==0.4.31 pytest-cov==2.5.1 pytest==3.1.2 sphinx==1.6.3 -aiohttp==2.2.2 +aiohttp==2.2.3 botocore==1.5.78 multidict==3.1.0 wrapt==1.10.10