From 3be383ccfdc906d220145ccf4ea6984740478570 Mon Sep 17 00:00:00 2001 From: Emad Mokhtar Date: Thu, 13 Feb 2020 08:44:31 +0100 Subject: [PATCH 1/3] Decode the error response if it is in byte format --- pydruid/client.py | 4 + tests/test_client.py | 178 ++++++++++++++++++++++++++----------------- 2 files changed, 114 insertions(+), 68 deletions(-) diff --git a/pydruid/client.py b/pydruid/client.py index 95dca1dd..70074af7 100755 --- a/pydruid/client.py +++ b/pydruid/client.py @@ -19,6 +19,7 @@ import json import re +from six import binary_type from six.moves import urllib from pydruid.query import QueryBuilder @@ -557,6 +558,9 @@ def _post(self, query): if e.code == 500: # has Druid returned an error? try: + if isinstance(err, binary_type): + # Decode the error before serialize it to JSON + err = err.decode("utf-8") err = json.loads(err) except ValueError: if HTML_ERROR.search(err): diff --git a/tests/test_client.py b/tests/test_client.py index 8f2d0a1b..c24e35fd 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,7 +4,7 @@ import pytest from mock import patch, Mock from six.moves import urllib -from six import StringIO +from six import StringIO, binary_type, BytesIO from pydruid.client import PyDruid from pydruid.query import Query @@ -17,23 +17,22 @@ def create_client(): def create_blank_query(): - return Query({}, 'none') + return Query({}, "none") -def _http_error(code, msg, data = ''): +def _http_error(code, msg, data=""): # Need a file-like object for the response data - fp = StringIO(data) + if isinstance(data, binary_type): + fp = BytesIO(data) + else: + fp = StringIO(data) return urllib.error.HTTPError( - url='http://fakeurl:8080/druid/v2/', - hdrs={}, - code=code, - msg=msg, - fp=fp, + url="http://fakeurl:8080/druid/v2/", hdrs={}, code=code, msg=msg, fp=fp, ) class TestPyDruid: - @patch('pydruid.client.urllib.request.urlopen') + @patch("pydruid.client.urllib.request.urlopen") def test_druid_returns_error(self, mock_urlopen): # given mock_urlopen.side_effect = _http_error(500, "Druid error") @@ -42,52 +41,63 @@ def test_druid_returns_error(self, mock_urlopen): # when / then with pytest.raises(IOError): client.topn( - datasource="testdatasource", - granularity="all", - intervals="2015-12-29/pt1h", - aggregations={"count": doublesum("count")}, - dimension="user_name", - metric="count", - filter=Dimension("user_lang") == "en", - threshold=1, - context={"timeout": 1000}) - - @patch('pydruid.client.urllib.request.urlopen') + datasource="testdatasource", + granularity="all", + intervals="2015-12-29/pt1h", + aggregations={"count": doublesum("count")}, + dimension="user_name", + metric="count", + filter=Dimension("user_lang") == "en", + threshold=1, + context={"timeout": 1000}, + ) + + @patch("pydruid.client.urllib.request.urlopen") def test_druid_returns_html_error(self, mock_urlopen): # given - message = textwrap.dedent(""" + message = textwrap.dedent( + """ - + Error 500

HTTP ERROR: 500

Problem accessing /druid/v2/. Reason: - 

    javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded

-
Powered by Jetty:// 9.3.19.v20170502
+ 
    javax.servlet.ServletException:
+            java.lang.OutOfMemoryError: GC overhead limit exceeded

+
+ Powered by Jetty:// 9.3.19.v20170502
- """).strip() - mock_urlopen.side_effect = _http_error(500, 'Internal Server Error', message) + """ + ).strip() + mock_urlopen.side_effect = _http_error(500, "Internal Server Error", message) client = create_client() # when / then with pytest.raises(IOError) as e: client.topn( - datasource="testdatasource", - granularity="all", - intervals="2015-12-29/pt1h", - aggregations={"count": doublesum("count")}, - dimension="user_name", - metric="count", - filter=Dimension("user_lang") == "en", - threshold=1, - context={"timeout": 1000}) - - assert str(e.value) == textwrap.dedent(""" - HTTP Error 500: Internal Server Error - Druid Error: javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded + datasource="testdatasource", + granularity="all", + intervals="2015-12-29/pt1h", + aggregations={"count": doublesum("count")}, + dimension="user_name", + metric="count", + filter=Dimension("user_lang") == "en", + threshold=1, + context={"timeout": 1000}, + ) + + assert ( + str(e.value) + == textwrap.dedent( + """ + HTTP Error 500: Internal Server Error + Druid Error: javax.servlet.ServletException: + java.lang.OutOfMemoryError: GC overhead limit exceeded Query is: { "aggregations": [ { @@ -112,9 +122,34 @@ def test_druid_returns_html_error(self, mock_urlopen): "queryType": "topN", "threshold": 1 } - """).strip() + """ + ).strip() + ) + + @patch("pydruid.client.urllib.request.urlopen") + def test_druid_returns_string_error_bytes_error_response(self, mock_urlopen): + # given + message = b"Error as bytes, please decode me" + mock_urlopen.side_effect = _http_error(500, "Internal Server Error", message) + client = create_client() - @patch('pydruid.client.urllib.request.urlopen') + # when / then + with pytest.raises(IOError) as e: + client.topn( + datasource="testdatasource", + granularity="all", + intervals="2015-12-29/pt1h", + aggregations={"count": doublesum("count")}, + dimension="user_name", + metric="count", + filter=Dimension("user_lang") == "en", + threshold=1, + context={"timeout": 1000}, + ) + + assert "Error as bytes, please decode me" in str(e.value) + + @patch("pydruid.client.urllib.request.urlopen") def test_druid_returns_results(self, mock_urlopen): # given response = Mock() @@ -126,28 +161,31 @@ def test_druid_returns_results(self, mock_urlopen): "metric" : 100 } ] } ] - """.encode("utf-8") + """.encode( + "utf-8" + ) mock_urlopen.return_value = response client = create_client() # when top = client.topn( - datasource="testdatasource", - granularity="all", - intervals="2015-12-29/pt1h", - aggregations={"count": doublesum("count")}, - dimension="user_name", - metric="count", - filter=Dimension("user_lang") == "en", - threshold=1, - context={"timeout": 1000}) + datasource="testdatasource", + granularity="all", + intervals="2015-12-29/pt1h", + aggregations={"count": doublesum("count")}, + dimension="user_name", + metric="count", + filter=Dimension("user_lang") == "en", + threshold=1, + context={"timeout": 1000}, + ) # then assert top is not None assert len(top.result) == 1 - assert len(top.result[0]['result']) == 1 + assert len(top.result[0]["result"]) == 1 - @patch('pydruid.client.urllib.request.urlopen') + @patch("pydruid.client.urllib.request.urlopen") def test_client_allows_to_export_last_query(self, mock_urlopen): # given response = Mock() @@ -159,29 +197,33 @@ def test_client_allows_to_export_last_query(self, mock_urlopen): "metric" : 100 } ] } ] - """.encode("utf-8") + """.encode( + "utf-8" + ) mock_urlopen.return_value = response client = create_client() client.topn( - datasource="testdatasource", - granularity="all", - intervals="2015-12-29/pt1h", - aggregations={"count": doublesum("count")}, - dimension="user_name", - metric="count", - filter=Dimension("user_lang") == "en", - threshold=1, - context={"timeout": 1000}) + datasource="testdatasource", + granularity="all", + intervals="2015-12-29/pt1h", + aggregations={"count": doublesum("count")}, + dimension="user_name", + metric="count", + filter=Dimension("user_lang") == "en", + threshold=1, + context={"timeout": 1000}, + ) # when / then - # assert that last_query.export_tsv method was called (it should throw an exception, given empty path) + # assert that last_query.export_tsv method was called (it should throw + # an exception, given empty path) with pytest.raises(TypeError): client.export_tsv(None) - @patch('pydruid.client.urllib.request.urlopen') + @patch("pydruid.client.urllib.request.urlopen") def test_client_auth_creds(self, mock_urlopen): client = create_client() query = create_blank_query() - client.set_basic_auth_credentials('myUsername', 'myPassword') + client.set_basic_auth_credentials("myUsername", "myPassword") headers, _, _ = client._prepare_url_headers_and_body(query) - assert headers['Authorization'] == "Basic bXlVc2VybmFtZTpteVBhc3N3b3Jk" + assert headers["Authorization"] == "Basic bXlVc2VybmFtZTpteVBhc3N3b3Jk" From aec8fe4a41c56ca93e7f367fcb72e58f563735d6 Mon Sep 17 00:00:00 2001 From: Emad Mokhtar Date: Sun, 21 Jun 2020 10:50:43 +0200 Subject: [PATCH 2/3] Add a config to ignore the E501:Too long line in the test_client file --- .flake8 | 3 +++ tests/test_client.py | 12 ++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.flake8 b/.flake8 index 3d4715b0..09963000 100644 --- a/.flake8 +++ b/.flake8 @@ -8,3 +8,6 @@ exclude = build import-order-style = google max-line-length = 90 + +per-file-ignores = + tests/test_client.py: E501 diff --git a/tests/test_client.py b/tests/test_client.py index c24e35fd..e236d1dc 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -59,17 +59,14 @@ def test_druid_returns_html_error(self, mock_urlopen): """ - + Error 500

HTTP ERROR: 500

Problem accessing /druid/v2/. Reason: - 

    javax.servlet.ServletException:
-            java.lang.OutOfMemoryError: GC overhead limit exceeded

-
- Powered by Jetty:// 9.3.19.v20170502
+ 
    javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded

+
Powered by Jetty:// 9.3.19.v20170502
""" @@ -96,8 +93,7 @@ def test_druid_returns_html_error(self, mock_urlopen): == textwrap.dedent( """ HTTP Error 500: Internal Server Error - Druid Error: javax.servlet.ServletException: - java.lang.OutOfMemoryError: GC overhead limit exceeded + Druid Error: javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded Query is: { "aggregations": [ { From 1093a8e006bdf5a34809d19743f6544064d93932 Mon Sep 17 00:00:00 2001 From: Emad Mokhtar Date: Sun, 5 Jul 2020 09:54:07 +0200 Subject: [PATCH 3/3] Fix isort and ignore the white tail in flake8 --- .flake8 | 2 +- tests/test_client.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index 09963000..e9589b0c 100644 --- a/.flake8 +++ b/.flake8 @@ -10,4 +10,4 @@ import-order-style = google max-line-length = 90 per-file-ignores = - tests/test_client.py: E501 + tests/test_client.py: E501 W291 diff --git a/tests/test_client.py b/tests/test_client.py index 44ab6907..1e16cd7f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,7 +1,7 @@ # -*- coding: UTF-8 -*- import textwrap import urllib -from io import StringIO, BytesIO +from io import BytesIO, StringIO from unittest.mock import Mock, patch import pytest @@ -92,8 +92,8 @@ def test_druid_returns_html_error(self, mock_urlopen): str(e.value) == textwrap.dedent( """ - HTTP Error 500: Internal Server Error - Druid Error: javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded + HTTP Error 500: Internal Server Error + Druid Error: javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded Query is: { "aggregations": [ {