From 4f49bf563e6ba3188ff5d7ab8015a34a71ea3f11 Mon Sep 17 00:00:00 2001 From: Markus Bong Date: Thu, 28 Jan 2021 15:26:03 +0100 Subject: [PATCH 1/4] add a semaphore to analyze to prevent http error 429 --- ssllabs/ssllabs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ssllabs/ssllabs.py b/ssllabs/ssllabs.py index 1091b10..bacb3ae 100644 --- a/ssllabs/ssllabs.py +++ b/ssllabs/ssllabs.py @@ -14,6 +14,7 @@ class Ssllabs(): def __init__(self): self.logger = logging.getLogger(f"{self.__class__.__module__}.{self.__class__.__name__}") + self._semaphore = asyncio.Semaphore(1) async def availability(self) -> bool: """ @@ -39,11 +40,20 @@ async def analyze(self, host: str, publish: bool = False, ignore_mismatch: bool See also: https://github.com/ssllabs/ssllabs-scan/blob/master/ssllabs-api-docs-v3.md#protocol-usage """ + await self._semaphore.acquire() + + i = Info() + info = await i.get() + if info.currentAssessments != 0: + await asyncio.sleep(info.newAssessmentCoolOff / 1000) + api = Analyze() host_object = await api.get(host=host, startNew="on", publish="on" if publish else "off", igonreMismatch="on" if ignore_mismatch else "off") + + self._semaphore.release() while host_object.status not in ["READY", "ERROR"]: self.logger.debug("Analyzing %s", host) await asyncio.sleep(10) From 1427237760ea4954b4ce8a0bbcf494fb2a0b5910 Mon Sep 17 00:00:00 2001 From: Markus Bong Date: Thu, 28 Jan 2021 15:26:03 +0100 Subject: [PATCH 2/4] add a semaphore to analyze to prevent http error 429 --- ssllabs/ssllabs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ssllabs/ssllabs.py b/ssllabs/ssllabs.py index 1091b10..bacb3ae 100644 --- a/ssllabs/ssllabs.py +++ b/ssllabs/ssllabs.py @@ -14,6 +14,7 @@ class Ssllabs(): def __init__(self): self.logger = logging.getLogger(f"{self.__class__.__module__}.{self.__class__.__name__}") + self._semaphore = asyncio.Semaphore(1) async def availability(self) -> bool: """ @@ -39,11 +40,20 @@ async def analyze(self, host: str, publish: bool = False, ignore_mismatch: bool See also: https://github.com/ssllabs/ssllabs-scan/blob/master/ssllabs-api-docs-v3.md#protocol-usage """ + await self._semaphore.acquire() + + i = Info() + info = await i.get() + if info.currentAssessments != 0: + await asyncio.sleep(info.newAssessmentCoolOff / 1000) + api = Analyze() host_object = await api.get(host=host, startNew="on", publish="on" if publish else "off", igonreMismatch="on" if ignore_mismatch else "off") + + self._semaphore.release() while host_object.status not in ["READY", "ERROR"]: self.logger.debug("Analyzing %s", host) await asyncio.sleep(10) From 1e9817867d1d10320b2d47bc2736b7bfe9d783dc Mon Sep 17 00:00:00 2001 From: Guido Schmitz Date: Thu, 28 Jan 2021 15:40:24 +0000 Subject: [PATCH 3/4] Respect maxAssessment --- ssllabs/ssllabs.py | 45 ++++++++++--------- tests/test_data/info_max_assessments.json | 10 +++++ tests/test_data/info_running_assessments.json | 10 +++++ tests/test_ssllabs.py | 33 ++++++++++++++ 4 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 tests/test_data/info_max_assessments.json create mode 100644 tests/test_data/info_running_assessments.json diff --git a/ssllabs/ssllabs.py b/ssllabs/ssllabs.py index bacb3ae..564b6d7 100644 --- a/ssllabs/ssllabs.py +++ b/ssllabs/ssllabs.py @@ -13,7 +13,7 @@ class Ssllabs(): """Highlevel methods to interact with the SSL Labs Assessment APIs.""" def __init__(self): - self.logger = logging.getLogger(f"{self.__class__.__module__}.{self.__class__.__name__}") + self._logger = logging.getLogger(f"{self.__class__.__module__}.{self.__class__.__name__}") self._semaphore = asyncio.Semaphore(1) async def availability(self) -> bool: @@ -22,17 +22,17 @@ async def availability(self) -> bool: See also: https://github.com/ssllabs/ssllabs-scan/blob/master/ssllabs-api-docs-v3.md#error-response-status-codes """ - api = Info() + i = Info() try: - await api.get() + await i.get() return True except HTTPStatusError as ex: - self.logger.error(ex) + self._logger.error(ex) return False async def analyze(self, host: str, publish: bool = False, ignore_mismatch: bool = False) -> HostData: """ - Test a particular host. + Test a particular host with respect to the cool off and the maximum number of assessments. :param host: Host to test :param publish: True if assessment results should be published on the public results boards @@ -41,23 +41,28 @@ async def analyze(self, host: str, publish: bool = False, ignore_mismatch: bool See also: https://github.com/ssllabs/ssllabs-scan/blob/master/ssllabs-api-docs-v3.md#protocol-usage """ await self._semaphore.acquire() - i = Info() info = await i.get() + + # Wait for a free slot, if all slots are in use + while info.currentAssessments >= info.maxAssessments: + await asyncio.sleep(1) + info = await i.get() + + # If there is already an assessment running, wait the needed cool off until starting the next one if info.currentAssessments != 0: await asyncio.sleep(info.newAssessmentCoolOff / 1000) - api = Analyze() - host_object = await api.get(host=host, - startNew="on", - publish="on" if publish else "off", - igonreMismatch="on" if ignore_mismatch else "off") - + a = Analyze() + host_object = await a.get(host=host, + startNew="on", + publish="on" if publish else "off", + igonreMismatch="on" if ignore_mismatch else "off") self._semaphore.release() while host_object.status not in ["READY", "ERROR"]: - self.logger.debug("Analyzing %s", host) + self._logger.debug("Analyzing %s", host) await asyncio.sleep(10) - host_object = await api.get(host=host, all="done") + host_object = await a.get(host=host, all="done") return host_object async def info(self) -> InfoData: @@ -66,8 +71,8 @@ async def info(self) -> InfoData: See also: https://github.com/ssllabs/ssllabs-scan/blob/master/ssllabs-api-docs-v3.md#info """ - api = Info() - return await api.get() + i = Info() + return await i.get() async def root_certs(self, trust_store: int = 1) -> str: """ @@ -80,8 +85,8 @@ async def root_certs(self, trust_store: int = 1) -> str: if not 1 <= trust_store <= 5: raise ValueError("""Trust store not found. Please choose on of the following: 1-Mozilla, 2-Apple MacOS, 3-Android, 4-Java, 5-Windows""") - api = RootCertsRaw() - return await api.get(trustStore=trust_store) + rcr = RootCertsRaw() + return await rcr.get(trustStore=trust_store) async def status_codes(self) -> StatusCodesData: """ @@ -89,5 +94,5 @@ async def status_codes(self) -> StatusCodesData: See also: https://github.com/ssllabs/ssllabs-scan/blob/master/ssllabs-api-docs-v3.md#retrieve-known-status-codes """ - api = StatusCodes() - return await api.get() + sc = StatusCodes() + return await sc.get() diff --git a/tests/test_data/info_max_assessments.json b/tests/test_data/info_max_assessments.json new file mode 100644 index 0000000..4c49d60 --- /dev/null +++ b/tests/test_data/info_max_assessments.json @@ -0,0 +1,10 @@ +{ + "engineVersion": "2.1.8", + "criteriaVersion": "2009q", + "maxAssessments": 25, + "currentAssessments": 25, + "newAssessmentCoolOff": 1000, + "messages": [ + "This assessment service is provided free of charge by Qualys SSL Labs, subject to our terms and conditions: https://www.ssllabs.com/about/terms.html" + ] +} diff --git a/tests/test_data/info_running_assessments.json b/tests/test_data/info_running_assessments.json new file mode 100644 index 0000000..4e20961 --- /dev/null +++ b/tests/test_data/info_running_assessments.json @@ -0,0 +1,10 @@ +{ + "engineVersion": "2.1.8", + "criteriaVersion": "2009q", + "maxAssessments": 25, + "currentAssessments": 1, + "newAssessmentCoolOff": 1000, + "messages": [ + "This assessment service is provided free of charge by Qualys SSL Labs, subject to our terms and conditions: https://www.ssllabs.com/about/terms.html" + ] +} diff --git a/tests/test_ssllabs.py b/tests/test_ssllabs.py index 5a9acd0..a691fe5 100644 --- a/tests/test_ssllabs.py +++ b/tests/test_ssllabs.py @@ -1,3 +1,4 @@ +import asyncio import dataclasses from unittest.mock import patch @@ -57,6 +58,38 @@ async def test_analyze_not_ready_yet(self, request, mocker): await ssllabs.analyze(host="devolo.de") assert spy.call_count == 2 + @pytest.mark.asyncio + async def test_analyze_max_assessments(self, request, mocker): + with patch("asyncio.sleep", new=AsyncMock()), \ + patch("ssllabs.api.analyze.Analyze.get", + new=AsyncMock(return_value=from_dict(data_class=HostData, + data=request.cls.analyze))), \ + patch("ssllabs.api.info.Info.get", + new=AsyncMock(side_effect=[ + from_dict(data_class=InfoData, + data=request.cls.info_max_assessments), + from_dict(data_class=InfoData, + data=request.cls.info) + ])): + spy = mocker.spy(asyncio, "sleep") + ssllabs = Ssllabs() + await ssllabs.analyze(host="devolo.de") + assert spy.call_count == 1 + + @pytest.mark.asyncio + async def test_analyze_running_assessments(self, request, mocker): + with patch("asyncio.sleep", new=AsyncMock()), \ + patch("ssllabs.api.analyze.Analyze.get", + new=AsyncMock(return_value=from_dict(data_class=HostData, + data=request.cls.analyze))), \ + patch("ssllabs.api.info.Info.get", + new=AsyncMock(return_value=from_dict(data_class=InfoData, + data=request.cls.info_running_assessments))): + spy = mocker.spy(asyncio, "sleep") + ssllabs = Ssllabs() + await ssllabs.analyze(host="devolo.de") + assert spy.call_count == 1 + @pytest.mark.asyncio async def test_root_certs(self, request): with patch("ssllabs.api.root_certs_raw.RootCertsRaw.get", From 36d9470ccdc5c70175a534eaf91d8087f1f90606 Mon Sep 17 00:00:00 2001 From: Guido Schmitz Date: Thu, 28 Jan 2021 15:46:16 +0000 Subject: [PATCH 4/4] Update version --- docs/CHANGELOG.md | 7 +++++++ ssllabs/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 383dc31..443b9ff 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,11 +4,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v0.2.0] - 2021/01/28 + +### Added + +- High level methods now respect cool off and maximum assessment rate limits + ## [v0.1.1] - 2021/01/27 ### Fixed - Fixed pip installation + ## [v0.1.0] - 2021/01/27 ### Added diff --git a/ssllabs/__init__.py b/ssllabs/__init__.py index 730ff46..fff4be1 100644 --- a/ssllabs/__init__.py +++ b/ssllabs/__init__.py @@ -1,6 +1,6 @@ from .ssllabs import Ssllabs __license__ = "MIT" -__version__ = "0.1.1" +__version__ = "0.2.0" __all__ = ['Ssllabs', "__license__", "__version__"]