Merge pull request #5 from GregEremeev/v_1

Version 1.0.0

Author: GregEremeev

MANIFEST.in

@@ -1,3 +1,4 @@
 include README.md
 include LICENSE.txt
 include requirements.txt
+include rosreestr_api/cacert.pem

README.md

@@ -11,7 +11,7 @@ pip install rosreestr-api
 
 2 Different ways how to get basic info about realty objects:
 ```python
-from rosreestr_api.clients import RosreestrAPIClient, AddressWrapper
+from rosreestr_api.clients.rosreestr import RosreestrAPIClient, AddressWrapper
 
 api_client = RosreestrAPIClient()
 
@@ -45,7 +45,7 @@ api_client.get_region_types(region_id=region_id)
 
 3 Different ways how to get geo info about realty objects:
 ```python
-from rosreestr_api.clients import PKKRosreestrAPIClient
+from rosreestr_api.clients.rosreestr import PKKRosreestrAPIClient
 
 api_client = PKKRosreestrAPIClient()
 

requirements.txt

@@ -1 +1,2 @@
-requests==2.22.0
+requests==2.31.0
+fake-useragent==1.5.0

rosreestr_api/cacert.pem

@@ -0,0 +1,157 @@
+import logging
+import os.path
+import ssl
+import time
+from typing import Union
+from urllib.parse import urlencode
+from importlib.util import find_spec
+
+import requests
+from requests import Session
+from requests.adapters import HTTPAdapter
+
+import rosreestr_api
+
+
+logger = logging.getLogger(__name__)
+
+
+class BaseHTTPClient:
+    SESSION_CLS = Session
+
+    GET_HTTP_METHOD = 'GET'
+    POST_HTTP_METHOD = 'POST'
+    PATCH_HTTP_METHOD = 'PATCH'
+    PUT_HTTP_METHOD = 'PUT'
+
+    BODY_LESS_METHODS = [GET_HTTP_METHOD]
+    LOG_REQUEST_TEMPLATE = '%(method)s %(url)s%(request_body)s%(duration)s'
+    LOG_RESPONSE_TEMPLATE = (LOG_REQUEST_TEMPLATE +
+                             ' - HTTP %(status_code)s%(response_body)s%(duration)s')
+
+    def __init__(self, timeout=3, keep_alive=False, default_headers=None):
+        self.timeout = timeout
+        self.keep_alive = keep_alive
+        self.default_headers = default_headers or {}
+        self._session = None
+
+    @property
+    def session(self) -> requests.Session:
+        if self.keep_alive:
+            if not self._session:
+                self._session = self.SESSION_CLS()
+            return self._session
+        else:
+            return self.SESSION_CLS()
+
+    def get(self, url, params=None, **kwargs) -> requests.Response:
+        if params:
+            url_with_query_params = url + '?' + urlencode(params)
+        else:
+            url_with_query_params = url
+
+        return self._make_request(self.GET_HTTP_METHOD, url_with_query_params, **kwargs)
+
+    def post(self, url, **kwargs) -> requests.Response:
+        return self._make_request(self.POST_HTTP_METHOD, url, **kwargs)
+
+    def patch(self, url, **kwargs) -> requests.Response:
+        return self._make_request(self.PATCH_HTTP_METHOD, url, **kwargs)
+
+    def put(self, url, **kwargs) -> requests.Response:
+        return self._make_request(self.PUT_HTTP_METHOD, url, **kwargs)
+
+    def _log_request(self, method, url, body, duration=None, log_method=logger.info):
+        message_params = {
+            'method': method, 'url': url, 'request_body': _get_body_for_logging(body),
+            'duration': _get_duration_for_logging(duration)}
+        log_method(self.LOG_REQUEST_TEMPLATE, message_params)
+
+    def _log_response(self, response, duration, log_method=logger.info):
+        message_params = {
+            'method': response.request.method,
+            'url': response.request.url,
+            'request_body': _get_body_for_logging(response.request.body),
+            'status_code': response.status_code,
+            'response_body': _get_body_for_logging(response.content),
+            'duration': _get_duration_for_logging(duration)}
+        log_method(self.LOG_RESPONSE_TEMPLATE, message_params)
+
+    def _make_request(self, method, url, **kwargs) -> requests.Response:
+        kwargs.setdefault('timeout', self.timeout)
+        session = self.session
+        timeout = kwargs.pop('timeout', self.timeout)
+
+        headers = self.default_headers.copy()
+        headers.update(kwargs.pop('headers', {}))
+
+        request = requests.Request(method, url, headers=headers, **kwargs)
+        prepared_request = request.prepare()
+        self._log_request(method, url, prepared_request.body)
+        start_time = time.time()
+        try:
+            response = session.send(prepared_request, timeout=timeout)
+            duration = time.time() - start_time
+            if response.status_code >= 400:
+                log_method = logging.error
+            else:
+                log_method = logging.debug
+
+            self._log_response(response, duration=duration, log_method=log_method)
+            return response
+        except requests.exceptions.RequestException as e:
+            duration = time.time() - start_time
+            if e.response:
+                self._log_response(e.response, duration=duration, log_method=logging.error)
+            else:
+                self._log_request(method, url, prepared_request.body, log_method=logging.exception)
+            raise
+        finally:
+            if not self.keep_alive:
+                session.close()
+
+
+class HTTPSAdapter(HTTPAdapter):
+    def init_poolmanager(self, *args, **kwargs):
+        ssl_context = ssl.create_default_context()
+        # https://www.openssl.org/docs/man3.0/man3/SSL_CTX_set_security_level.html
+        # rosreestr supports only SECLEVEL 1
+        ssl_context.set_ciphers('DEFAULT@SECLEVEL=1')
+        # We want to use the most secured protocol from security level 1
+        ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2
+        kwargs['ssl_context'] = ssl_context
+        return super().init_poolmanager(*args, **kwargs)
+
+
+class CustomSession(Session):
+    CACERT_PATH = os.path.join(
+        os.path.dirname(find_spec(rosreestr_api.__name__).origin),
+        'cacert.pem'
+    )
+    def __init__(self):
+        super().__init__()
+        self.verify = self.CACERT_PATH
+        self.mount(prefix='https://', adapter=HTTPSAdapter())
+
+
+class RosreestrHTTPClient(BaseHTTPClient):
+    SESSION_CLS = CustomSession
+
+
+def _get_body_for_logging(body: Union[bytes, str]) -> str:
+    try:
+        if isinstance(body, bytes):
+            return (b' BODY: ' + body).decode('utf-8')
+        elif isinstance(body, str):
+            return ' BODY: ' + body
+        else:
+            return ''
+    except UnicodeDecodeError:
+        return ''
+
+
+def _get_duration_for_logging(duration: str) -> str:
+    if duration is not None:
+        return ' {0:.6f}s'.format(duration)
+    else:
+        return ''

rosreestr_api/tests/__init__.py renamed to rosreestr_api/clients/__init__.py

@@ -1,137 +1,15 @@
-import time
 import logging
-from typing import Union
 from dataclasses import dataclass
-from urllib.parse import urlencode, quote_plus
+from urllib.parse import quote_plus
 
 import requests
+from fake_useragent import UserAgent
 
+from rosreestr_api.clients.http import RosreestrHTTPClient
 
 logger = logging.getLogger(__name__)
 
 
-def _strip_cadastral_id(cadastral_id):
-    stripped_cadastral_id = []
-    cadastral_id = cadastral_id.split(':')
-    for part in cadastral_id:
-        if part:
-            stripped_cadastral_id.append(part[:-1].lstrip('0') + part[-1])
-    return ':'.join(stripped_cadastral_id)
-
-
-def _get_body_for_logging(body: Union[bytes, str]) -> str:
-    try:
-        if isinstance(body, bytes):
-            return (b' BODY: ' + body).decode('utf-8')
-        elif isinstance(body, str):
-            return ' BODY: ' + body
-        else:
-            return ''
-    except UnicodeDecodeError:
-        return ''
-
-
-def _get_duration_for_logging(duration: str) -> str:
-    if duration is not None:
-        return ' {0:.6f}s'.format(duration)
-    else:
-        return ''
-
-
-class HTTPClient:
-
-    GET_HTTP_METHOD = 'GET'
-    POST_HTTP_METHOD = 'POST'
-    PATCH_HTTP_METHOD = 'PATCH'
-    PUT_HTTP_METHOD = 'PUT'
-
-    BODY_LESS_METHODS = [GET_HTTP_METHOD]
-    LOG_REQUEST_TEMPLATE = '%(method)s %(url)s%(request_body)s%(duration)s'
-    LOG_RESPONSE_TEMPLATE = (LOG_REQUEST_TEMPLATE +
-                             ' - HTTP %(status_code)s%(response_body)s%(duration)s')
-
-    def __init__(self, timeout=3, keep_alive=False, default_headers=None):
-        self.timeout = timeout
-        self.keep_alive = keep_alive
-        self.default_headers = default_headers or {}
-        self._session = None
-
-    def _log_request(self, method, url, body, duration=None, log_method=logger.info):
-        message_params = {
-            'method': method, 'url': url, 'request_body': _get_body_for_logging(body),
-            'duration': _get_duration_for_logging(duration)}
-        log_method(self.LOG_REQUEST_TEMPLATE, message_params)
-
-    def _log_response(self, response, duration, log_method=logger.info):
-        message_params = {
-            'method': response.request.method,
-            'url': response.request.url,
-            'request_body': _get_body_for_logging(response.request.body),
-            'status_code': response.status_code,
-            'response_body': _get_body_for_logging(response.content),
-            'duration': _get_duration_for_logging(duration)}
-        log_method(self.LOG_RESPONSE_TEMPLATE, message_params)
-
-    def _make_request(self, method, url, **kwargs) -> requests.Response:
-        kwargs.setdefault('timeout', self.timeout)
-        session = self.session
-        timeout = kwargs.pop('timeout', self.timeout)
-
-        headers = self.default_headers.copy()
-        headers.update(kwargs.pop('headers', {}))
-
-        request = requests.Request(method, url, headers=headers, **kwargs)
-        prepared_request = request.prepare()
-        self._log_request(method, url, prepared_request.body)
-        start_time = time.time()
-        try:
-            response = session.send(prepared_request, timeout=timeout)
-            duration = time.time() - start_time
-            if response.status_code >= 400:
-                log_method = logging.error
-            else:
-                log_method = logging.debug
-
-            self._log_response(response, duration=duration, log_method=log_method)
-            return response
-        except requests.exceptions.RequestException as e:
-            duration = time.time() - start_time
-            if e.response:
-                self._log_response(e.response, duration=duration, log_method=logging.error)
-            else:
-                self._log_request(method, url, prepared_request.body, log_method=logging.exception)
-            raise
-        finally:
-            if not self.keep_alive:
-                session.close()
-
-    @property
-    def session(self) -> requests.Session:
-        if self.keep_alive:
-            if not self._session:
-                self._session = requests.Session()
-            return self._session
-        else:
-            return requests.Session()
-
-    def get(self, url, params=None, **kwargs) -> requests.Response:
-        if params:
-            url_with_query_params = url + '?' + urlencode(params)
-        else:
-            url_with_query_params = url
-
-        return self._make_request(self.GET_HTTP_METHOD, url_with_query_params, **kwargs)
-
-    def post(self, url, **kwargs) -> requests.Response:
-        return self._make_request(self.POST_HTTP_METHOD, url, **kwargs)
-
-    def patch(self, url, **kwargs) -> requests.Response:
-        return self._make_request(self.PATCH_HTTP_METHOD, url, **kwargs)
-
-    def put(self, url, **kwargs) -> requests.Response:
-        return self._make_request(self.PUT_HTTP_METHOD, url, **kwargs)
-
-
 @dataclass
 class AddressWrapper:
 
@@ -156,7 +34,7 @@ def __post_init__(self):
 
 class RosreestrAPIClient:
 
-    BASE_URL = 'http://rosreestr.ru/api/online'
+    BASE_URL = 'https://rosreestr.gov.ru/api/online'
     MACRO_REGIONS_URL = f'{BASE_URL}/macro_regions/'
     REGIONS_URL = f'{BASE_URL}/regions/' + '{}/'
     REGION_TYPES_URL = f'{BASE_URL}/region_types/' + '{}/'
@@ -171,7 +49,11 @@ class RosreestrAPIClient:
     REPUBLIC = 'республика'
 
     def __init__(self, timeout=5, keep_alive=False):
-        self._http_client = HTTPClient(timeout=timeout, keep_alive=keep_alive)
+        self._http_client = RosreestrHTTPClient(
+            timeout=timeout,
+            keep_alive=keep_alive,
+            default_headers={'User-Agent': UserAgent().random}
+        )
         self._macro_regions = None
         self._macro_regions_to_regions = None
 
@@ -296,7 +178,11 @@ class PKKRosreestrAPIClient:
     SEARCH_PARCEL_BY_CADASTRAL_ID_URL = SEARCH_OBJECT_BY_CADASTRAL_ID.format(object_type=1)
 
     def __init__(self, timeout=5, keep_alive=False):
-        self._http_client = HTTPClient(timeout=timeout, keep_alive=keep_alive)
+        self._http_client = RosreestrHTTPClient(
+            timeout=timeout,
+            keep_alive=keep_alive,
+            default_headers={'User-Agent': UserAgent().random}
+        )
 
     def get_parcel_by_coordinates(self, *, lat, long, limit=11, tolerance=2) -> dict:
         url = self.SEARCH_PARCEL_BY_COORDINATES_URL.format(
@@ -317,3 +203,12 @@ def get_building_by_coordinates(self, *, lat, long, limit=11, tolerance=2) -> di
         url = self.SEARCH_BUILDING_BY_COORDINATES_URL.format(
             lat=lat, long=long, limit=limit, tolerance=tolerance)
         return self._http_client.get(url).json()
+
+
+def _strip_cadastral_id(cadastral_id):
+    stripped_cadastral_id = []
+    cadastral_id = cadastral_id.split(':')
+    for part in cadastral_id:
+        if part:
+            stripped_cadastral_id.append(part[:-1].lstrip('0') + part[-1])
+    return ':'.join(stripped_cadastral_id)

rosreestr_api/clients/http.py

@@ -8,17 +8,22 @@
     name='rosreestr-api',
     author='Greg Eremeev',
     author_email='gregory.eremeev@gmail.com',
-    version='0.3.4',
+    version='1.0.0',
     license='BSD-3-Clause',
     url='https://github.com/GregEremeev/rosreestr-api',
     install_requires=requirements,
-    description='Toolset to work with rosreestr.ru/api',
+    description='Toolset to work with rosreestr.gov.ru/api and pkk.rosreestr.ru/api',
     packages=find_packages(),
     extras_require={'dev': ['ipdb>=0.13.2', 'pytest>=5.4.1', 'httpretty>=1.0.2']},
     classifiers=[
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Programming Language :: Python :: Implementation :: CPython'
     ],
     zip_safe=False,