diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index f7be5f0..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,60 +0,0 @@ -version: 2.0 - -jobs: - "python-2.7": - docker: - - image: circleci/python:2.7 - steps: - - checkout - - restore_cache: - key: py27-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-test.txt" }} - - run: - command: | - virtualenv venv - . venv/bin/activate - pip install -r requirements-test.txt - - save_cache: - key: py27-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-test.txt" }} - paths: - - "venv" - - run: - command: | - . venv/bin/activate - mkdir -p /tmp/results - py.test -v -s --junitxml=/tmp/results/pytest.xml tests/ - - store_artifacts: - path: /tmp/results - destination: python-2.7 - - "python-3.7": - docker: - - image: circleci/python:3.7 - steps: - - checkout - - restore_cache: - key: py37-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-test.txt" }} - - run: - command: | - python -m venv venv - . venv/bin/activate - pip install -r requirements-test.txt - - save_cache: - key: py37-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-test.txt" }} - paths: - - "venv" - - run: - command: | - . venv/bin/activate - mkdir -p /tmp/results - mypy --py2 --ignore-missing-imports --junit-xml=/tmp/results/mypy.xml popget - py.test -v -s --junitxml=/tmp/results/pytest.xml tests/ - - store_artifacts: - path: /tmp/results - destination: python-3.7 - -workflows: - version: 2 - build: - jobs: - - "python-2.7" - - "python-3.7" diff --git a/Makefile b/Makefile index f850a7d..01f6039 100644 --- a/Makefile +++ b/Makefile @@ -11,14 +11,11 @@ tag: git push --tags mypy: - mypy --py2 --ignore-missing-imports popget + mypy --ignore-missing-imports popget pytest: py.test -v -s tests/ -pytest-pdb: - py.test -v -s --ipdb tests/ - test: $(MAKE) mypy $(MAKE) pytest diff --git a/README.rst b/README.rst index 2d785d2..06cb2a3 100644 --- a/README.rst +++ b/README.rst @@ -31,21 +31,14 @@ as your existing Django ``settings.py``. To bootstrap this, there are a couple of env vars to control how config is loaded: -- ``POPGET_APP_CONFIG`` - should be an import path to a python module, for example: - ``POPGET_APP_CONFIG=django.conf.settings`` -- ``POPGET_CONFIG_NAMESPACE`` - Sets the prefix used for loading further config values from env and - config file. Defaults to ``POPGET``. - -See source of ``popget/conf/defaults.py`` for more details. +See source of ``popget/conf/settings.py`` for more details. Some useful config keys (all of which are prefixed with ``POPGET_`` by default): -- ``_CLIENT_DEFAULT_USER_AGENT`` when making requests, popget will use this +- ``POPGET_CLIENT_DEFAULT_USER_AGENT`` when making requests, popget will use this string as the user agent. -- ``_CLIENT_TIMEOUT`` if ``None`` then no timeout, otherwise this timeout +- ``POPGET_CLIENT_TIMEOUT`` if ``None`` then no timeout, otherwise this timeout (in seconds) will be applied to all requests. Requests which timeout will return a 504 response, which will be raised as an ``HTTPError``. @@ -313,32 +306,12 @@ Compatibility This project is tested against: =========== === -Python 2.7 * -Python 3.7 * +Python 3.11 * =========== === Running the tests ----------------- -CircleCI -~~~~~~~~ - -| The easiest way to test the full version matrix is to install the - CircleCI command line app: -| https://circleci.com/docs/2.0/local-jobs/ -| (requires Docker) - -The cli does not support 'workflows' at the moment so you have to run -the two Python version jobs separately: - -.. code:: bash - - circleci build --job python-2.7 - -.. code:: bash - - circleci build --job python-3.7 - py.test (single python version) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -349,6 +322,6 @@ Decide which Python version you want to test and create a virtualenv: .. code:: bash - pyenv virtualenv 3.7.4 popget + python -m virtualenv .venv -p python3.11 pip install -r requirements-test.txt - py.test -v -s --ipdb tests/ + py.test -v -s tests/ diff --git a/popget/__about__.py b/popget/__about__.py index cfe6447..afced14 100644 --- a/popget/__about__.py +++ b/popget/__about__.py @@ -1 +1 @@ -__version__ = '1.8.3' +__version__ = '2.0.0' diff --git a/popget/__init__.py b/popget/__init__.py index cf135db..fd0db89 100644 --- a/popget/__init__.py +++ b/popget/__init__.py @@ -1,5 +1,5 @@ -from popget.client import APIClient # noqa -from popget.endpoint import ( # noqa +from popget.client import APIClient +from popget.endpoint import ( Arg, BodyType, DeleteEndpoint, diff --git a/popget/client.py b/popget/client.py index 11b5f6d..e1b4302 100644 --- a/popget/client.py +++ b/popget/client.py @@ -1,31 +1,18 @@ from functools import partial from six import add_metaclass -from typing import Any, Callable, Dict, Optional, Set, Tuple, Type, TypeVar, Union # noqa +from typing import Any, Callable, Type -from mypy_extensions import Arg, DefaultArg, KwArg import requests from requests.exceptions import Timeout from popget.conf import settings from popget.endpoint import APIEndpoint, BodyType, BODY_CONTENT_TYPES, NO_DEFAULT from popget.errors import MissingRequiredArg -from popget.extratypes import ResponseTypes # noqa +from popget.extratypes import ResponseTypes from popget.utils import get_base_attr, update_nested -ClientMethod = Callable[ - [ - Arg(Type['APIClient'], 'cls'), - DefaultArg(Optional[Dict[Any, Any]], '_request_kwargs'), - DefaultArg(Optional[requests.Session], '_session'), - KwArg(object), - ], - Union[ResponseTypes, object] -] - - -def method_factory(endpoint, client_method_name): - # type: (APIEndpoint, str) -> ClientMethod +def method_factory(endpoint: APIEndpoint, client_method_name: str): """ Kwargs: endpoint: the endpoint to generate a callable method for @@ -36,8 +23,9 @@ def method_factory(endpoint, client_method_name): In turn the method returns response content, either string or deserialized JSON data. """ - def _prepare_request(base_url, _request_kwargs=None, **call_kwargs): - # type: (str, Optional[Dict], **Any) -> Tuple[str, Dict[str, Dict]] + def _prepare_request( + base_url: str, _request_kwargs: dict | None = None, **call_kwargs + ) -> tuple[str, dict[str, dict]]: """ Kwargs: base_url: base url of API @@ -100,12 +88,12 @@ def _prepare_request(base_url, _request_kwargs=None, **call_kwargs): return url, request_kwargs - def client_method(cls, # type: Type[APIClient] - _request_kwargs=None, # type: Optional[Dict] - _session=None, # type: requests.Session - **call_kwargs - ): - # type: (...) -> Union[ResponseTypes, object] + def client_method( + cls, + _request_kwargs: dict | None = None, + _session: requests.Session | None = None, + **call_kwargs + ) -> ResponseTypes | object: """ Returns: Response... for non-async clients this will be response content, @@ -126,17 +114,16 @@ def client_method(cls, # type: Type[APIClient] class ConfigClass(object): - base_url = None # type: str - session_cls = None # type: Type[requests.Session] - _session = None # type: requests.Session + base_url: str + session_cls: Type[requests.Session] + _session: requests.Session | None = None def __init__(self, config): self.base_url = getattr(config, 'base_url', '') self.session_cls = getattr(config, 'session_cls', requests.Session) @property - def session(self): - # type: () -> requests.Session + def session(self) -> requests.Session: if not self._session: session = self.session_cls() session.headers['User-Agent'] = settings.CLIENT_DEFAULT_USER_AGENT @@ -179,8 +166,9 @@ class Config: config_class = ConfigClass @staticmethod - def add_methods_for_endpoint(methods, name, endpoint, config): - # type: (Dict[str, classmethod], str, APIEndpoint, Any) -> None + def add_methods_for_endpoint( + methods: dict[str, classmethod], name: str, endpoint: APIEndpoint, config: Any + ) -> None: methods[name] = classmethod(method_factory(endpoint, '_make_request')) def __new__(cls, name, bases, attrs): @@ -198,8 +186,7 @@ def __new__(cls, name, bases, attrs): return type.__new__(cls, name, bases, attrs) -def get_response_body(response): - # type: (requests.Response) -> ResponseTypes +def get_response_body(response: requests.Response) -> ResponseTypes: """ Kwargs: response: from requests lib @@ -231,12 +218,12 @@ def get_response_body(response): @add_metaclass(APIClientMetaclass) class APIClient(object): - _config = None # type: ConfigClass + _config: ConfigClass @staticmethod - def _request_kwargs(method, url, args, kwargs): - # type: (str, str, Tuple[Any, ...], Dict[str, Any]) -> None - + def _request_kwargs( + method: str, url: str, args: tuple[Any, ...], kwargs: dict[str, Any] + ) -> None: if settings.CLIENT_DISABLE_VERIFY_SSL: kwargs['verify'] = False @@ -245,9 +232,7 @@ def _request_kwargs(method, url, args, kwargs): kwargs['timeout'] = settings.CLIENT_TIMEOUT @staticmethod - def handle(call, request_url): - # type: (Callable[[], requests.Response], str) -> ResponseTypes - + def handle(call: Callable[[], requests.Response], request_url: str) -> ResponseTypes: try: res = call() except Timeout as e: @@ -262,8 +247,9 @@ def handle(call, request_url): return get_response_body(res) @classmethod - def _make_request(cls, method, url, session=None, *args, **kwargs): - # type: (str, str, Optional[requests.Session], *Any, **Any) -> ResponseTypes + def _make_request( + cls, method: str, url: str, session: requests.Session | None = None, *args, **kwargs + ) -> ResponseTypes: """ Don't call this directly. Instead, add APIEndpoint instances to your APIClient sub-class definition. Accessor methods will be generated by diff --git a/popget/conf/__init__.py b/popget/conf/__init__.py index 481d314..e69de29 100644 --- a/popget/conf/__init__.py +++ b/popget/conf/__init__.py @@ -1,4 +0,0 @@ -from flexisettings import Settings - - -settings = Settings('POPGET', 'popget.conf.defaults') diff --git a/popget/conf/defaults.py b/popget/conf/defaults.py deleted file mode 100644 index 3f3b9c5..0000000 --- a/popget/conf/defaults.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -from typing import Optional - -from popget.__about__ import __version__ - - -# namespace for config keys loaded from e.g. Django conf or env vars -CONFIG_NAMESPACE = os.getenv('POPGET_CONFIG_NAMESPACE', 'POPGET') - -# optional import path to file containing namespaced config (e.g. 'django.conf.settings') -APP_CONFIG = os.getenv('POPGET_APP_CONFIG', None) - - -CLIENT_DEFAULT_USER_AGENT = 'popget/{}'.format(__version__) - -CLIENT_TIMEOUT = None # type: Optional[float] - -CLIENT_DISABLE_VERIFY_SSL = False diff --git a/popget/conf/settings.py b/popget/conf/settings.py new file mode 100644 index 0000000..37d91d2 --- /dev/null +++ b/popget/conf/settings.py @@ -0,0 +1,14 @@ +from popget.__about__ import __version__ + +try: + from django.conf import settings +except ImportError: + settings = None + +CLIENT_DEFAULT_USER_AGENT: str = getattr( + settings, 'POPGET_CLIENT_DEFAULT_USER_AGENT', 'popget/{}'.format(__version__) +) + +CLIENT_TIMEOUT: float = getattr(settings, 'POPGET_CLIENT_TIMEOUT', 3.0) + +CLIENT_DISABLE_VERIFY_SSL: bool = getattr(settings, 'POPGET_CLIENT_DISABLE_VERIFY_SSL', False) diff --git a/popget/endpoint.py b/popget/endpoint.py index 14841b0..eb44477 100644 --- a/popget/endpoint.py +++ b/popget/endpoint.py @@ -1,10 +1,9 @@ import string -from typing import Any, Callable, Dict, Optional, Set, Tuple, Type, Union # noqa +from typing import Any, Callable, Type from enum import Enum # via enum34 package on python 2.7 from popget.errors import ArgNameConflict -from popget.extratypes import ResponseTypes # noqa class BodyType(Enum): @@ -25,8 +24,7 @@ class BodyType(Enum): } -def _validate_arg_name(arg, arg_type, reserved): - # type: (str, str, Set[str]) -> str +def _validate_arg_name(arg: str, arg_type: str, reserved: set[str]) -> str: """ Validate that `arg` does not conflict with names already defined in `reserved`. @@ -58,8 +56,9 @@ class Arg(object): Querystring argument """ - def __init__(self, name, required=False, default=NO_DEFAULT): - # type: (str, bool, Union[object, Callable]) -> None + def __init__( + self, name: str, required: bool = False, default: object | Callable = NO_DEFAULT + ) -> None: self.name = name self.required = required if required and default != NO_DEFAULT: @@ -152,29 +151,29 @@ class ThingServiceClient(APIClient): RESERVED_NAMES = ('_request_kwargs', '_session') - method = None # type: str - path = None # type: str - - body_type = None # type: str - body_arg = None # type: str - - request_headers = None # type: Optional[Dict[str, str]] - - required_args = None # type: Set[str] - url_args = None # type: Set[str] - querystring_args = None # type: Set[Arg] - request_header_args = None # type: Dict[str, Set[str]] - - def __init__(self, - method, # type: str - path, # type: str - querystring_args=None, # type: Optional[Tuple[Arg, ...]] - request_headers=None, # type: Optional[Dict[str, str]] - body_type=BodyType.JSON, # type: BodyType - body_arg='body', # type: str - body_required=False, # type: bool - ): - # type: (...) -> None + method: str + path: str + + body_type: str + body_arg: str + + request_headers: dict[str, str] | None = None + + required_args: set[str] + url_args: set[str] + querystring_args: set[Arg] + request_header_args: dict[str, set[str]] + + def __init__( + self, + method: str, + path: str, + querystring_args: tuple[Arg, ...] | None = None, + request_headers: dict[str, str] | None = None, + body_type: BodyType = BodyType.JSON, + body_arg: str = 'body', + body_required: bool = False, + ) -> None: """ Kwargs: method: 'GET', 'POST' etc @@ -230,7 +229,7 @@ def __init__(self, querystring_args_.add(arg) # parse request-header args - request_header_args = {} # type: Dict[str, Set[str]] + request_header_args: dict[str, set[str]] = {} if request_headers is not None: for header, value in request_headers.items(): for tokens in f.parse(value): diff --git a/popget/extratypes.py b/popget/extratypes.py index c4ca40d..92363a9 100644 --- a/popget/extratypes.py +++ b/popget/extratypes.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Union +from typing import Any _T = Any # should be 'JSONTypes' but mypy doesn't support recursive types yet -JSONTypes = Union[Dict[str, _T], List[_T], str, float, bool, None] +JSONTypes = dict[str, _T] | list[_T] | str | float | bool | None -JSONObject = Dict[str, JSONTypes] +JSONObject = dict[str, JSONTypes] -JSONList = List[JSONTypes] +JSONList = list[JSONTypes] -ResponseTypes = Union[bytes, JSONTypes] +ResponseTypes = bytes | JSONTypes diff --git a/popget/nonblocking/threadpool.py b/popget/nonblocking/threadpool.py index 5e79ef0..2d7b9d8 100644 --- a/popget/nonblocking/threadpool.py +++ b/popget/nonblocking/threadpool.py @@ -1,9 +1,9 @@ -import concurrent.futures # noqa +import concurrent.futures from functools import partial from six import add_metaclass -from typing import Any, Dict, Optional, Type # noqa +from typing import Type, Any -import requests # noqa +import requests from requests_futures.sessions import FuturesSession from popget.client import ( @@ -12,14 +12,13 @@ ConfigClass as BaseConfigClass, method_factory, ) -from popget.endpoint import APIEndpoint # noqa -from popget.extratypes import ResponseTypes # noqa +from popget.endpoint import APIEndpoint from popget.utils import classproperty class ConfigClass(BaseConfigClass): - _async_session = None # type: FuturesSession + _async_session: FuturesSession = None async_method_template = 'async_{}' @@ -28,8 +27,7 @@ def __init__(self, config): super(ConfigClass, self).__init__(config) @classproperty - def async_session(cls): - # type: () -> FuturesSession + def async_session(cls) -> FuturesSession: if not cls._async_session: cls._async_session = FuturesSession(session=cls.session) return cls._async_session @@ -40,8 +38,7 @@ class AsyncAPIClientMetaclass(APIClientMetaclass): config_class = ConfigClass @staticmethod - def add_methods_for_endpoint(methods, name, endpoint, config): - # type: (Dict[str, classmethod], str, APIEndpoint, Any) -> None + def add_methods_for_endpoint(methods: dict[str, classmethod], name: str, endpoint: APIEndpoint, config: Any) -> None: methods[name] = classmethod(method_factory(endpoint, '_make_request')) methods[config.async_method_template.format(name)] = classmethod( method_factory(endpoint, '_make_async_request') @@ -51,11 +48,12 @@ def add_methods_for_endpoint(methods, name, endpoint, config): @add_metaclass(AsyncAPIClientMetaclass) class APIClient(BaseAPIClient): - _config = None # type: ConfigClass + _config: ConfigClass @classmethod - def _make_async_request(cls, method, url, session=None, *args, **kwargs): - # type: (str, str, Optional[requests.Session], *Any, **Any) -> concurrent.futures.Future + def _make_async_request( + cls, method: str, url: str, session: requests.Session | None = None, *args, **kwargs + ) -> concurrent.futures.Future: """ Don't call this directly. Instead, add APIEndpoint instances to your APIClient sub-class definition. Accessor methods will be generated by diff --git a/popget/utils.py b/popget/utils.py index cb431e4..e7e68f9 100644 --- a/popget/utils.py +++ b/popget/utils.py @@ -1,9 +1,9 @@ import collections -from typing import AnyStr, Dict, Optional, Tuple, Type +from types import MappingProxyType +from typing import Any -def update_nested(d, u): - # type: (Dict, collections.Mapping) -> Dict +def update_nested(d: dict, u: collections.abc.Mapping) -> dict: """ https://stackoverflow.com/a/3233356/202168 A dict update that supports nested keys without overwriting the whole @@ -26,7 +26,7 @@ def update_nested(d, u): } """ for k, v in u.items(): - if isinstance(v, collections.Mapping): + if isinstance(v, collections.abc.Mapping): r = update_nested(d.get(k, {}), v) d[k] = r else: @@ -34,8 +34,7 @@ def update_nested(d, u): return d -def get_base_attr(attr, bases, attrs): - # type: (str, Tuple[Type, ...], Dict[AnyStr, object]) -> Optional[object] +def get_base_attr(attr: str, bases: tuple[type, ...], attrs: dict[str, Any] | MappingProxyType) -> object | None: """ Given an attr name, recursively look through the given base classes until finding one where that attr is present, returning the value. diff --git a/requirements-dev.txt b/requirements-dev.txt index deef4a8..9de50cf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,5 +2,4 @@ ipython ipdb -robpol86-pytest-ipdb twine diff --git a/requirements-test.txt b/requirements-test.txt index 2f8891c..19dc024 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,7 @@ -r requirements.txt pytest -mypy==0.782; python_version >= '3.4' responses +mypy +types-requests +types-six diff --git a/requirements.txt b/requirements.txt index 9fb6199..47a3158 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,6 @@ -flexisettings>=1.0,<1.1 -requests>2.0,<3.0 -typing>=3.6.2,<4.0; python_version < '3.6' -six -enum34>=1.1.6,<2.0.0; python_version < '3.4' -futures>=3.1.1,<3.2.0; python_version < '3.2' -mypy_extensions>=0.3,<0.5 +requests<3.0.0 +six<2.0.0 +enum34<2.0.0 +futures<4.0.0 -requests-futures==0.9.7 +requests-futures~=1.0.0 diff --git a/setup.py b/setup.py index 41c2192..d7e525e 100644 --- a/setup.py +++ b/setup.py @@ -41,19 +41,15 @@ 'Programming Language :: Python', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.11', ], install_requires=[ - 'flexisettings>=1.0,<1.1', - 'requests>2.0,<3.0', - 'typing>=3.6.2,<4.0; python_version < "3.6"', - 'six', - 'enum34>=1.1.6,<2.0.0; python_version < "3.4"', - 'futures>=3.1.1,<3.2.0; python_version < "3.2"', - 'mypy_extensions>=0.3,<0.5', + 'requests<3.0.0', + 'six<2.0.0', + 'enum34<2.0.0', + 'futures<4.0.0', + 'requests-futures>=0.9.7,<1.0.0', ], - extras_require={ - 'threadpool': ['requests-futures>=0.9.7,<1.0.0'], - }, packages=[ 'popget', diff --git a/tests/test_client.py b/tests/test_client.py index 23e738f..e0d067c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,6 +1,9 @@ +from unittest.mock import patch + +from _pytest import fixtures +from _pytest.fixtures import fixture from six.moves.urllib import parse as urlparse -from flexisettings.utils import override_settings from urllib3.exceptions import ConnectTimeoutError import pytest import requests @@ -512,12 +515,14 @@ class Config: base_url = 'http://10.255.255.1/' -@override_settings(settings, CLIENT_TIMEOUT=1) def test_timeout(): """ Test APIClient behaviour when the requests library timeout threshold is reached """ - with pytest.raises(requests.exceptions.HTTPError) as exc_info: + with ( + pytest.raises(requests.exceptions.HTTPError) as exc_info, + patch('popget.client.settings.CLIENT_TIMEOUT', 1.0) + ): TimeoutService.thing_detail(id=777) e = exc_info.value