Skip to content

Commit

Permalink
Decode the error response if it is in byte format
Browse files Browse the repository at this point in the history
  • Loading branch information
@EmadMokhtar
EmadMokhtar committed Feb 13, 2020
1 parent 438e30d commit 3be383c
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 68 deletions.
4 changes: 4 additions & 0 deletions pydruid/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import json
import re

from six import binary_type
from six.moves import urllib

from pydruid.query import QueryBuilder
Expand Down Expand Up @@ -557,6 +558,9 @@ def _post(self, query):
if e.code == 500:
# has Druid returned an error?
try:
if isinstance(err, binary_type):
# Decode the error before serialize it to JSON
err = err.decode("utf-8")
err = json.loads(err)
except ValueError:
if HTML_ERROR.search(err):
Expand Down
178 changes: 110 additions & 68 deletions tests/test_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import pytest
from mock import patch, Mock
from six.moves import urllib
from six import StringIO
from six import StringIO, binary_type, BytesIO

from pydruid.client import PyDruid
from pydruid.query import Query
Expand All @@ -17,23 +17,22 @@ def create_client():


def create_blank_query():
return Query({}, 'none')
return Query({}, "none")


def _http_error(code, msg, data = ''):
def _http_error(code, msg, data=""):
# Need a file-like object for the response data
fp = StringIO(data)
if isinstance(data, binary_type):
fp = BytesIO(data)
else:
fp = StringIO(data)
return urllib.error.HTTPError(
url='http://fakeurl:8080/druid/v2/',
hdrs={},
code=code,
msg=msg,
fp=fp,
url="http://fakeurl:8080/druid/v2/", hdrs={}, code=code, msg=msg, fp=fp,
)


class TestPyDruid:
@patch('pydruid.client.urllib.request.urlopen')
@patch("pydruid.client.urllib.request.urlopen")
def test_druid_returns_error(self, mock_urlopen):
# given
mock_urlopen.side_effect = _http_error(500, "Druid error")
Expand All @@ -42,52 +41,63 @@ def test_druid_returns_error(self, mock_urlopen):
# when / then
with pytest.raises(IOError):
client.topn(
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000})

@patch('pydruid.client.urllib.request.urlopen')
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000},
)

@patch("pydruid.client.urllib.request.urlopen")
def test_druid_returns_html_error(self, mock_urlopen):
# given
message = textwrap.dedent("""
message = textwrap.dedent(
"""
<html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<meta http-equiv="Content-Type" content="text/html;
charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /druid/v2/. Reason:
<pre> javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.19.v20170502</a><hr/>
<pre> javax.servlet.ServletException:
java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
<hr /><a href="http://eclipse.org/jetty">
Powered by Jetty:// 9.3.19.v20170502</a><hr/>
</body>
</html>
""").strip()
mock_urlopen.side_effect = _http_error(500, 'Internal Server Error', message)
"""
).strip()
mock_urlopen.side_effect = _http_error(500, "Internal Server Error", message)
client = create_client()

# when / then
with pytest.raises(IOError) as e:
client.topn(
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000})

assert str(e.value) == textwrap.dedent("""
HTTP Error 500: Internal Server Error
Druid Error: javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000},
)

assert (
str(e.value)
== textwrap.dedent(
"""
HTTP Error 500: Internal Server Error
Druid Error: javax.servlet.ServletException:
java.lang.OutOfMemoryError: GC overhead limit exceeded
Query is: {
"aggregations": [
{
Expand All @@ -112,9 +122,34 @@ def test_druid_returns_html_error(self, mock_urlopen):
"queryType": "topN",
"threshold": 1
}
""").strip()
"""
).strip()
)

@patch("pydruid.client.urllib.request.urlopen")
def test_druid_returns_string_error_bytes_error_response(self, mock_urlopen):
# given
message = b"Error as bytes, please decode me"
mock_urlopen.side_effect = _http_error(500, "Internal Server Error", message)
client = create_client()

@patch('pydruid.client.urllib.request.urlopen')
# when / then
with pytest.raises(IOError) as e:
client.topn(
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000},
)

assert "Error as bytes, please decode me" in str(e.value)

@patch("pydruid.client.urllib.request.urlopen")
def test_druid_returns_results(self, mock_urlopen):
# given
response = Mock()
Expand All @@ -126,28 +161,31 @@ def test_druid_returns_results(self, mock_urlopen):
"metric" : 100
} ]
} ]
""".encode("utf-8")
""".encode(
"utf-8"
)
mock_urlopen.return_value = response
client = create_client()

# when
top = client.topn(
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000})
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000},
)

# then
assert top is not None
assert len(top.result) == 1
assert len(top.result[0]['result']) == 1
assert len(top.result[0]["result"]) == 1

@patch('pydruid.client.urllib.request.urlopen')
@patch("pydruid.client.urllib.request.urlopen")
def test_client_allows_to_export_last_query(self, mock_urlopen):
# given
response = Mock()
Expand All @@ -159,29 +197,33 @@ def test_client_allows_to_export_last_query(self, mock_urlopen):
"metric" : 100
} ]
} ]
""".encode("utf-8")
""".encode(
"utf-8"
)
mock_urlopen.return_value = response
client = create_client()
client.topn(
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000})
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000},
)

# when / then
# assert that last_query.export_tsv method was called (it should throw an exception, given empty path)
# assert that last_query.export_tsv method was called (it should throw
# an exception, given empty path)
with pytest.raises(TypeError):
client.export_tsv(None)

@patch('pydruid.client.urllib.request.urlopen')
@patch("pydruid.client.urllib.request.urlopen")
def test_client_auth_creds(self, mock_urlopen):
client = create_client()
query = create_blank_query()
client.set_basic_auth_credentials('myUsername', 'myPassword')
client.set_basic_auth_credentials("myUsername", "myPassword")
headers, _, _ = client._prepare_url_headers_and_body(query)
assert headers['Authorization'] == "Basic bXlVc2VybmFtZTpteVBhc3N3b3Jk"
assert headers["Authorization"] == "Basic bXlVc2VybmFtZTpteVBhc3N3b3Jk"

0 comments on commit 3be383c

Please sign in to comment.